From 892a0d44dc9581df1d739094fdf24b96867079f9 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 8 Feb 2017 08:13:17 -0500 Subject: [PATCH 001/519] Initial commit --- packages/pandas-gbq/.gitignore | 102 +++++++++++++++++++++++++++++++++ packages/pandas-gbq/README.md | 1 + 2 files changed, 103 insertions(+) create mode 100644 packages/pandas-gbq/.gitignore create mode 100644 packages/pandas-gbq/README.md diff --git a/packages/pandas-gbq/.gitignore b/packages/pandas-gbq/.gitignore new file mode 100644 index 000000000000..a77e780f3332 --- /dev/null +++ b/packages/pandas-gbq/.gitignore @@ -0,0 +1,102 @@ +######################################### +# Editor temporary/working/backup files # +.#* +*\#*\# +[#]*# +*~ +*$ +*.bak +*flymake* +*.kdev4 +*.log +*.swp +*.pdb +.project +.pydevproject +.settings +.idea +.vagrant +.noseids +.ipynb_checkpoints +.tags + +# Compiled source # +################### +*.a +*.com +*.class +*.dll +*.exe +*.pxi +*.o +*.py[ocd] +*.so +.build_cache_dir +MANIFEST + +# Python files # +################ +# setup.py working directory +build +# sphinx build directory +doc/_build +# setup.py dist directory +dist +# Egg metadata +*.egg-info +.eggs +.pypirc + +# tox testing tool +.tox +# rope +.ropeproject +# wheel files +*.whl +**/wheelhouse/* +# coverage +.coverage + +# OS generated files # +###################### +.directory +.gdb_history +.DS_Store +ehthumbs.db +Icon? +Thumbs.db + +# Data files # +############## +*.dta +*.xpt +*.h5 +pandas/io/*.dat +pandas/io/*.json +scikits + +# Generated Sources # +##################### +!skts.c +!np_datetime.c +!np_datetime_strings.c +*.c +*.cpp + +# Performance Testing # +####################### +asv_bench/env/ +asv_bench/html/ +asv_bench/results/ +asv_bench/pandas/ + +# Documentation generated files # +################################# +doc/source/generated +doc/source/_static +doc/source/vbench +doc/source/vbench.rst +doc/source/index.rst +doc/build/html/index.html +# Windows specific leftover: +doc/tmp.sv diff --git a/packages/pandas-gbq/README.md b/packages/pandas-gbq/README.md new file mode 100644 index 000000000000..1d895546aeb2 --- /dev/null +++ b/packages/pandas-gbq/README.md @@ -0,0 +1 @@ +**pandas-gbq** is a package providing an interface to the Google Big Query interface from pandas From b01f98028f38b7bfdf1c86ea3150b7ea4076bb55 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 8 Feb 2017 08:14:07 -0500 Subject: [PATCH 002/519] fix .gitignore --- packages/pandas-gbq/.gitignore | 37 +--------------------------------- 1 file changed, 1 insertion(+), 36 deletions(-) diff --git a/packages/pandas-gbq/.gitignore b/packages/pandas-gbq/.gitignore index a77e780f3332..eac15860886d 100644 --- a/packages/pandas-gbq/.gitignore +++ b/packages/pandas-gbq/.gitignore @@ -1,4 +1,4 @@ -######################################### +gi######################################### # Editor temporary/working/backup files # .#* *\#*\# @@ -65,38 +65,3 @@ dist ehthumbs.db Icon? Thumbs.db - -# Data files # -############## -*.dta -*.xpt -*.h5 -pandas/io/*.dat -pandas/io/*.json -scikits - -# Generated Sources # -##################### -!skts.c -!np_datetime.c -!np_datetime_strings.c -*.c -*.cpp - -# Performance Testing # -####################### -asv_bench/env/ -asv_bench/html/ -asv_bench/results/ -asv_bench/pandas/ - -# Documentation generated files # -################################# -doc/source/generated -doc/source/_static -doc/source/vbench -doc/source/vbench.rst -doc/source/index.rst -doc/build/html/index.html -# Windows specific leftover: -doc/tmp.sv From 871bc1b8dc670a4db481bd3e777d5c95155ae4ef Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 8 Feb 2017 08:28:05 -0500 Subject: [PATCH 003/519] add setup, versioning, recipes --- packages/pandas-gbq/.travis.yml | 33 + packages/pandas-gbq/LICENSE.md | 87 + packages/pandas-gbq/README.md | 1 - packages/pandas-gbq/README.rst | 29 + packages/pandas-gbq/conda.recipe/bld.bat | 8 + packages/pandas-gbq/conda.recipe/build.sh | 9 + packages/pandas-gbq/conda.recipe/meta.yaml | 27 + packages/pandas-gbq/pandas_gbq/__init__.py | 4 + .../pandas-gbq/pandas_gbq/tests/__init__.py | 4 + packages/pandas-gbq/setup.cfg | 15 + packages/pandas-gbq/setup.py | 52 + packages/pandas-gbq/versioneer.py | 1699 +++++++++++++++++ 12 files changed, 1967 insertions(+), 1 deletion(-) create mode 100644 packages/pandas-gbq/.travis.yml create mode 100644 packages/pandas-gbq/LICENSE.md delete mode 100644 packages/pandas-gbq/README.md create mode 100644 packages/pandas-gbq/README.rst create mode 100644 packages/pandas-gbq/conda.recipe/bld.bat create mode 100644 packages/pandas-gbq/conda.recipe/build.sh create mode 100644 packages/pandas-gbq/conda.recipe/meta.yaml create mode 100644 packages/pandas-gbq/pandas_gbq/__init__.py create mode 100644 packages/pandas-gbq/pandas_gbq/tests/__init__.py create mode 100644 packages/pandas-gbq/setup.cfg create mode 100644 packages/pandas-gbq/setup.py create mode 100644 packages/pandas-gbq/versioneer.py diff --git a/packages/pandas-gbq/.travis.yml b/packages/pandas-gbq/.travis.yml new file mode 100644 index 000000000000..bfed5a8acfeb --- /dev/null +++ b/packages/pandas-gbq/.travis.yml @@ -0,0 +1,33 @@ +sudo: false + +language: python + +env: + - PYTHON=2.7 PANDAS=0.19.2 + - PYTHON=3.4 PANDAS=0.18.1 + - PYTHON=3.5 PANDAS=0.19.2 + - PYTHON=3.6 PANDAS=0.19.2 + +install: + - wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; + - bash miniconda.sh -b -p $HOME/miniconda + - export PATH="$HOME/miniconda/bin:$PATH" + - hash -r + - conda config --set always_yes yes --set changeps1 no + - conda config --add channels pandas + - conda update -q conda + + # Useful for debugging any issues with conda + - conda info -a + - conda create -q -n test-environment python=$PYTHON pandas=$PANDAS pytest flake8 coverage setuptools + - pip install httplib2 google-api-python-client oauth2client pytest-cov + - source activate test-environment + - conda list + - python setup.py install + +script: + - pytest pandas_gbq + - flake8 --version + +after_success: + - coveralls diff --git a/packages/pandas-gbq/LICENSE.md b/packages/pandas-gbq/LICENSE.md new file mode 100644 index 000000000000..474cd65bfb48 --- /dev/null +++ b/packages/pandas-gbq/LICENSE.md @@ -0,0 +1,87 @@ +======= +License +======= + +pandas is distributed under a 3-clause ("Simplified" or "New") BSD +license. Parts of NumPy, SciPy, numpydoc, bottleneck, which all have +BSD-compatible licenses, are included. Their licenses follow the pandas +license. + +pandas license +============== + +Copyright (c) 2011-2012, Lambda Foundry, Inc. and PyData Development Team +All rights reserved. + +Copyright (c) 2008-2011 AQR Capital Management, LLC +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the copyright holder nor the names of any + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +About the Copyright Holders +=========================== + +AQR Capital Management began pandas development in 2008. Development was +led by Wes McKinney. AQR released the source under this license in 2009. +Wes is now an employee of Lambda Foundry, and remains the pandas project +lead. + +The PyData Development Team is the collection of developers of the PyData +project. This includes all of the PyData sub-projects, including pandas. The +core team that coordinates development on GitHub can be found here: +http://github.com/pydata. + +Full credits for pandas contributors can be found in the documentation. + +Our Copyright Policy +==================== + +PyData uses a shared copyright model. Each contributor maintains copyright +over their contributions to PyData. However, it is important to note that +these contributions are typically only changes to the repositories. Thus, +the PyData source code, in its entirety, is not the copyright of any single +person or institution. Instead, it is the collective copyright of the +entire PyData Development Team. If individual contributors want to maintain +a record of what changes/contributions they have specific copyright on, +they should indicate their copyright in the commit message of the change +when they commit the change to one of the PyData repositories. + +With this in mind, the following banner should be used in any source code +file to indicate the copyright and license terms: + +#----------------------------------------------------------------------------- +# Copyright (c) 2012, PyData Development Team +# All rights reserved. +# +# Distributed under the terms of the BSD Simplified License. +# +# The full license is in the LICENSE file, distributed with this software. +#----------------------------------------------------------------------------- + +Other licenses can be found in the LICENSES directory. diff --git a/packages/pandas-gbq/README.md b/packages/pandas-gbq/README.md deleted file mode 100644 index 1d895546aeb2..000000000000 --- a/packages/pandas-gbq/README.md +++ /dev/null @@ -1 +0,0 @@ -**pandas-gbq** is a package providing an interface to the Google Big Query interface from pandas diff --git a/packages/pandas-gbq/README.rst b/packages/pandas-gbq/README.rst new file mode 100644 index 000000000000..b83c15bb97ee --- /dev/null +++ b/packages/pandas-gbq/README.rst @@ -0,0 +1,29 @@ +pandas-gbq +========== + +**pandas-gbq** is a package providing an interface to the Google Big Query interface from pandas + + +Installation +------------ + + +Install latest release version via pip +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: shell + + $ pip install pandas-gbq + +Install latest development version +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: shell + + $ pip install git+https://github.com/pydata/pandas-gbq.git + + +Usage +----- + +See the `pandas-gbq documentation `_ for more details. diff --git a/packages/pandas-gbq/conda.recipe/bld.bat b/packages/pandas-gbq/conda.recipe/bld.bat new file mode 100644 index 000000000000..87b1481d740c --- /dev/null +++ b/packages/pandas-gbq/conda.recipe/bld.bat @@ -0,0 +1,8 @@ +"%PYTHON%" setup.py install +if errorlevel 1 exit 1 + +:: Add more build steps here, if they are necessary. + +:: See +:: http://docs.continuum.io/conda/build.html +:: for a list of environment variables that are set during the build process. diff --git a/packages/pandas-gbq/conda.recipe/build.sh b/packages/pandas-gbq/conda.recipe/build.sh new file mode 100644 index 000000000000..4d7fc032b8cb --- /dev/null +++ b/packages/pandas-gbq/conda.recipe/build.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +$PYTHON setup.py install + +# Add more build steps here, if they are necessary. + +# See +# http://docs.continuum.io/conda/build.html +# for a list of environment variables that are set during the build process. diff --git a/packages/pandas-gbq/conda.recipe/meta.yaml b/packages/pandas-gbq/conda.recipe/meta.yaml new file mode 100644 index 000000000000..57ba6095378e --- /dev/null +++ b/packages/pandas-gbq/conda.recipe/meta.yaml @@ -0,0 +1,27 @@ +package: + name: pandas-gbq + version: "0.1.0" + +source: + git_url: ../ + +requirements: + build: + - python + - setuptools + + run: + - python + - pandas + - httplib2 + - google-api-python-client + - oauth2client + +test: + # Python imports + imports: + - pandas_gbq +about: + home: https://github.com/pydata/pandas-gbq + license: BSD License + summary: 'pandas-gbq is a package providing an interface to the Google Big Query interface from pandas' diff --git a/packages/pandas-gbq/pandas_gbq/__init__.py b/packages/pandas-gbq/pandas_gbq/__init__.py new file mode 100644 index 000000000000..40c60425991c --- /dev/null +++ b/packages/pandas-gbq/pandas_gbq/__init__.py @@ -0,0 +1,4 @@ +__version__ = version = '0.3.0.post' + +from .data import (get_components_yahoo, get_data_famafrench, get_data_google, get_data_yahoo, get_data_enigma, # noqa + get_data_yahoo_actions, get_quote_google, get_quote_yahoo, DataReader, Options) # noqa diff --git a/packages/pandas-gbq/pandas_gbq/tests/__init__.py b/packages/pandas-gbq/pandas_gbq/tests/__init__.py new file mode 100644 index 000000000000..40c60425991c --- /dev/null +++ b/packages/pandas-gbq/pandas_gbq/tests/__init__.py @@ -0,0 +1,4 @@ +__version__ = version = '0.3.0.post' + +from .data import (get_components_yahoo, get_data_famafrench, get_data_google, get_data_yahoo, get_data_enigma, # noqa + get_data_yahoo_actions, get_quote_google, get_quote_yahoo, DataReader, Options) # noqa diff --git a/packages/pandas-gbq/setup.cfg b/packages/pandas-gbq/setup.cfg new file mode 100644 index 000000000000..cab8f5b77750 --- /dev/null +++ b/packages/pandas-gbq/setup.cfg @@ -0,0 +1,15 @@ + +# See the docstring in versioneer.py for instructions. Note that you must +# re-run 'versioneer.py setup' after changing this section, and commit the +# resulting files. + +[versioneer] +VCS = git +style = pep440 +versionfile_source = pandas_gbq/_version.py +versionfile_build = pandas_gbq/_version.py +tag_prefix = v +parentdir_prefix = pandas_gbq- + +[flake8] +ignore = E731 diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py new file mode 100644 index 000000000000..d9d19642d007 --- /dev/null +++ b/packages/pandas-gbq/setup.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from ast import parse +import os +from setuptools import setup, find_packages + + +NAME = 'pandas-gbq' + + +# versioning +import versioneer +cmdclass = versioneer.get_cmdclass() + +def readme(): + with open('README.rst') as f: + return f.read() + +INSTALL_REQUIRES = ( + ['pandas', 'requests>=2.3.0', 'requests-file', 'requests-ftp'] +) + +setup( + name=NAME, + version=versioneer.get_version(), + description="Pandas interface to Google Big Query", + long_description=readme(), + license='BSD License', + author='The PyData Development Team', + author_email='pydata@googlegroups.com', + url='https://github.com/pydata/pandas-gbq', + classifiers=[ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: Science/Research', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Topic :: Scientific/Engineering', + ], + keywords='data', + install_requires=INSTALL_REQUIRES, + packages=find_packages(exclude=['contrib', 'docs', 'tests*']), + test_suite='tests', + zip_safe=False, +) diff --git a/packages/pandas-gbq/versioneer.py b/packages/pandas-gbq/versioneer.py new file mode 100644 index 000000000000..c010f63e3ead --- /dev/null +++ b/packages/pandas-gbq/versioneer.py @@ -0,0 +1,1699 @@ + +# Version: 0.15 + +""" +The Versioneer +============== + +* like a rocketeer, but for versions! +* https://github.com/warner/python-versioneer +* Brian Warner +* License: Public Domain +* Compatible With: python2.6, 2.7, 3.2, 3.3, 3.4, and pypy +* [![Latest Version] +(https://pypip.in/version/versioneer/badge.svg?style=flat) +](https://pypi.python.org/pypi/versioneer/) +* [![Build Status] +(https://travis-ci.org/warner/python-versioneer.png?branch=master) +](https://travis-ci.org/warner/python-versioneer) + +This is a tool for managing a recorded version number in distutils-based +python projects. The goal is to remove the tedious and error-prone "update +the embedded version string" step from your release process. Making a new +release should be as easy as recording a new tag in your version-control +system, and maybe making new tarballs. + + +## Quick Install + +* `pip install versioneer` to somewhere to your $PATH +* add a `[versioneer]` section to your setup.cfg (see below) +* run `versioneer install` in your source tree, commit the results + +## Version Identifiers + +Source trees come from a variety of places: + +* a version-control system checkout (mostly used by developers) +* a nightly tarball, produced by build automation +* a snapshot tarball, produced by a web-based VCS browser, like github's + "tarball from tag" feature +* a release tarball, produced by "setup.py sdist", distributed through PyPI + +Within each source tree, the version identifier (either a string or a number, +this tool is format-agnostic) can come from a variety of places: + +* ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows + about recent "tags" and an absolute revision-id +* the name of the directory into which the tarball was unpacked +* an expanded VCS keyword ($Id$, etc) +* a `_version.py` created by some earlier build step + +For released software, the version identifier is closely related to a VCS +tag. Some projects use tag names that include more than just the version +string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool +needs to strip the tag prefix to extract the version identifier. For +unreleased software (between tags), the version identifier should provide +enough information to help developers recreate the same tree, while also +giving them an idea of roughly how old the tree is (after version 1.2, before +version 1.3). Many VCS systems can report a description that captures this, +for example `git describe --tags --dirty --always` reports things like +"0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the +0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has +uncommitted changes. + +The version identifier is used for multiple purposes: + +* to allow the module to self-identify its version: `myproject.__version__` +* to choose a name and prefix for a 'setup.py sdist' tarball + +## Theory of Operation + +Versioneer works by adding a special `_version.py` file into your source +tree, where your `__init__.py` can import it. This `_version.py` knows how to +dynamically ask the VCS tool for version information at import time. + +`_version.py` also contains `$Revision$` markers, and the installation +process marks `_version.py` to have this marker rewritten with a tag name +during the `git archive` command. As a result, generated tarballs will +contain enough information to get the proper version. + +To allow `setup.py` to compute a version too, a `versioneer.py` is added to +the top level of your source tree, next to `setup.py` and the `setup.cfg` +that configures it. This overrides several distutils/setuptools commands to +compute the version when invoked, and changes `setup.py build` and `setup.py +sdist` to replace `_version.py` with a small static file that contains just +the generated version data. + +## Installation + +First, decide on values for the following configuration variables: + +* `VCS`: the version control system you use. Currently accepts "git". + +* `style`: the style of version string to be produced. See "Styles" below for + details. Defaults to "pep440", which looks like + `TAG[+DISTANCE.gSHORTHASH[.dirty]]`. + +* `versionfile_source`: + + A project-relative pathname into which the generated version strings should + be written. This is usually a `_version.py` next to your project's main + `__init__.py` file, so it can be imported at runtime. If your project uses + `src/myproject/__init__.py`, this should be `src/myproject/_version.py`. + This file should be checked in to your VCS as usual: the copy created below + by `setup.py setup_versioneer` will include code that parses expanded VCS + keywords in generated tarballs. The 'build' and 'sdist' commands will + replace it with a copy that has just the calculated version string. + + This must be set even if your project does not have any modules (and will + therefore never import `_version.py`), since "setup.py sdist" -based trees + still need somewhere to record the pre-calculated version strings. Anywhere + in the source tree should do. If there is a `__init__.py` next to your + `_version.py`, the `setup.py setup_versioneer` command (described below) + will append some `__version__`-setting assignments, if they aren't already + present. + +* `versionfile_build`: + + Like `versionfile_source`, but relative to the build directory instead of + the source directory. These will differ when your setup.py uses + 'package_dir='. If you have `package_dir={'myproject': 'src/myproject'}`, + then you will probably have `versionfile_build='myproject/_version.py'` and + `versionfile_source='src/myproject/_version.py'`. + + If this is set to None, then `setup.py build` will not attempt to rewrite + any `_version.py` in the built tree. If your project does not have any + libraries (e.g. if it only builds a script), then you should use + `versionfile_build = None` and override `distutils.command.build_scripts` + to explicitly insert a copy of `versioneer.get_version()` into your + generated script. + +* `tag_prefix`: + + a string, like 'PROJECTNAME-', which appears at the start of all VCS tags. + If your tags look like 'myproject-1.2.0', then you should use + tag_prefix='myproject-'. If you use unprefixed tags like '1.2.0', this + should be an empty string. + +* `parentdir_prefix`: + + a optional string, frequently the same as tag_prefix, which appears at the + start of all unpacked tarball filenames. If your tarball unpacks into + 'myproject-1.2.0', this should be 'myproject-'. To disable this feature, + just omit the field from your `setup.cfg`. + +This tool provides one script, named `versioneer`. That script has one mode, +"install", which writes a copy of `versioneer.py` into the current directory +and runs `versioneer.py setup` to finish the installation. + +To versioneer-enable your project: + +* 1: Modify your `setup.cfg`, adding a section named `[versioneer]` and + populating it with the configuration values you decided earlier (note that + the option names are not case-sensitive): + + ```` + [versioneer] + VCS = git + style = pep440 + versionfile_source = src/myproject/_version.py + versionfile_build = myproject/_version.py + tag_prefix = "" + parentdir_prefix = myproject- + ```` + +* 2: Run `versioneer install`. This will do the following: + + * copy `versioneer.py` into the top of your source tree + * create `_version.py` in the right place (`versionfile_source`) + * modify your `__init__.py` (if one exists next to `_version.py`) to define + `__version__` (by calling a function from `_version.py`) + * modify your `MANIFEST.in` to include both `versioneer.py` and the + generated `_version.py` in sdist tarballs + + `versioneer install` will complain about any problems it finds with your + `setup.py` or `setup.cfg`. Run it multiple times until you have fixed all + the problems. + +* 3: add a `import versioneer` to your setup.py, and add the following + arguments to the setup() call: + + version=versioneer.get_version(), + cmdclass=versioneer.get_cmdclass(), + +* 4: commit these changes to your VCS. To make sure you won't forget, + `versioneer install` will mark everything it touched for addition using + `git add`. Don't forget to add `setup.py` and `setup.cfg` too. + +## Post-Installation Usage + +Once established, all uses of your tree from a VCS checkout should get the +current version string. All generated tarballs should include an embedded +version string (so users who unpack them will not need a VCS tool installed). + +If you distribute your project through PyPI, then the release process should +boil down to two steps: + +* 1: git tag 1.0 +* 2: python setup.py register sdist upload + +If you distribute it through github (i.e. users use github to generate +tarballs with `git archive`), the process is: + +* 1: git tag 1.0 +* 2: git push; git push --tags + +Versioneer will report "0+untagged.NUMCOMMITS.gHASH" until your tree has at +least one tag in its history. + +## Version-String Flavors + +Code which uses Versioneer can learn about its version string at runtime by +importing `_version` from your main `__init__.py` file and running the +`get_versions()` function. From the "outside" (e.g. in `setup.py`), you can +import the top-level `versioneer.py` and run `get_versions()`. + +Both functions return a dictionary with different flavors of version +information: + +* `['version']`: A condensed version string, rendered using the selected + style. This is the most commonly used value for the project's version + string. The default "pep440" style yields strings like `0.11`, + `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section + below for alternative styles. + +* `['full-revisionid']`: detailed revision identifier. For Git, this is the + full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac". + +* `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that + this is only accurate if run in a VCS checkout, otherwise it is likely to + be False or None + +* `['error']`: if the version string could not be computed, this will be set + to a string describing the problem, otherwise it will be None. It may be + useful to throw an exception in setup.py if this is set, to avoid e.g. + creating tarballs with a version string of "unknown". + +Some variants are more useful than others. Including `full-revisionid` in a +bug report should allow developers to reconstruct the exact code being tested +(or indicate the presence of local changes that should be shared with the +developers). `version` is suitable for display in an "about" box or a CLI +`--version` output: it can be easily compared against release notes and lists +of bugs fixed in various releases. + +The installer adds the following text to your `__init__.py` to place a basic +version in `YOURPROJECT.__version__`: + + from ._version import get_versions + __version__ = get_versions()['version'] + del get_versions + +## Styles + +The setup.cfg `style=` configuration controls how the VCS information is +rendered into a version string. + +The default style, "pep440", produces a PEP440-compliant string, equal to the +un-prefixed tag name for actual releases, and containing an additional "local +version" section with more detail for in-between builds. For Git, this is +TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags +--dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the +tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and +that this commit is two revisions ("+2") beyond the "0.11" tag. For released +software (exactly equal to a known tag), the identifier will only contain the +stripped tag, e.g. "0.11". + +Other styles are available. See details.md in the Versioneer source tree for +descriptions. + +## Debugging + +Versioneer tries to avoid fatal errors: if something goes wrong, it will tend +to return a version of "0+unknown". To investigate the problem, run `setup.py +version`, which will run the version-lookup code in a verbose mode, and will +display the full contents of `get_versions()` (including the `error` string, +which may help identify what went wrong). + +## Updating Versioneer + +To upgrade your project to a new release of Versioneer, do the following: + +* install the new Versioneer (`pip install -U versioneer` or equivalent) +* edit `setup.cfg`, if necessary, to include any new configuration settings + indicated by the release notes +* re-run `versioneer install` in your source tree, to replace + `SRC/_version.py` +* commit any changed files + +### Upgrading to 0.15 + +Starting with this version, Versioneer is configured with a `[versioneer]` +section in your `setup.cfg` file. Earlier versions required the `setup.py` to +set attributes on the `versioneer` module immediately after import. The new +version will refuse to run (raising an exception during import) until you +have provided the necessary `setup.cfg` section. + +In addition, the Versioneer package provides an executable named +`versioneer`, and the installation process is driven by running `versioneer +install`. In 0.14 and earlier, the executable was named +`versioneer-installer` and was run without an argument. + +### Upgrading to 0.14 + +0.14 changes the format of the version string. 0.13 and earlier used +hyphen-separated strings like "0.11-2-g1076c97-dirty". 0.14 and beyond use a +plus-separated "local version" section strings, with dot-separated +components, like "0.11+2.g1076c97". PEP440-strict tools did not like the old +format, but should be ok with the new one. + +### Upgrading from 0.11 to 0.12 + +Nothing special. + +### Upgrading from 0.10 to 0.11 + +You must add a `versioneer.VCS = "git"` to your `setup.py` before re-running +`setup.py setup_versioneer`. This will enable the use of additional +version-control systems (SVN, etc) in the future. + +## Future Directions + +This tool is designed to make it easily extended to other version-control +systems: all VCS-specific components are in separate directories like +src/git/ . The top-level `versioneer.py` script is assembled from these +components by running make-versioneer.py . In the future, make-versioneer.py +will take a VCS name as an argument, and will construct a version of +`versioneer.py` that is specific to the given VCS. It might also take the +configuration arguments that are currently provided manually during +installation by editing setup.py . Alternatively, it might go the other +direction and include code from all supported VCS systems, reducing the +number of intermediate scripts. + + +## License + +To make Versioneer easier to embed, all its code is hereby released into the +public domain. The `_version.py` that it creates is also in the public +domain. + +""" + +from __future__ import print_function +try: + import configparser +except ImportError: + import ConfigParser as configparser +import errno +import json +import os +import re +import subprocess +import sys + + +class VersioneerConfig: + pass + + +def get_root(): + # we require that all commands are run from the project root, i.e. the + # directory that contains setup.py, setup.cfg, and versioneer.py . + root = os.path.realpath(os.path.abspath(os.getcwd())) + setup_py = os.path.join(root, "setup.py") + versioneer_py = os.path.join(root, "versioneer.py") + if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): + # allow 'python path/to/setup.py COMMAND' + root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) + setup_py = os.path.join(root, "setup.py") + versioneer_py = os.path.join(root, "versioneer.py") + if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): + err = ("Versioneer was unable to run the project root directory. " + "Versioneer requires setup.py to be executed from " + "its immediate directory (like 'python setup.py COMMAND'), " + "or in a way that lets it use sys.argv[0] to find the root " + "(like 'python path/to/setup.py COMMAND').") + raise VersioneerBadRootError(err) + try: + # Certain runtime workflows (setup.py install/develop in a setuptools + # tree) execute all dependencies in a single python process, so + # "versioneer" may be imported multiple times, and python's shared + # module-import table will cache the first one. So we can't use + # os.path.dirname(__file__), as that will find whichever + # versioneer.py was first imported, even in later projects. + me = os.path.realpath(os.path.abspath(__file__)) + if os.path.splitext(me)[0] != os.path.splitext(versioneer_py)[0]: + print("Warning: build in %s is using versioneer.py from %s" + % (os.path.dirname(me), versioneer_py)) + except NameError: + pass + return root + + +def get_config_from_root(root): + # This might raise EnvironmentError (if setup.cfg is missing), or + # configparser.NoSectionError (if it lacks a [versioneer] section), or + # configparser.NoOptionError (if it lacks "VCS="). See the docstring at + # the top of versioneer.py for instructions on writing your setup.cfg . + setup_cfg = os.path.join(root, "setup.cfg") + parser = configparser.SafeConfigParser() + with open(setup_cfg, "r") as f: + parser.readfp(f) + VCS = parser.get("versioneer", "VCS") # mandatory + + def get(parser, name): + if parser.has_option("versioneer", name): + return parser.get("versioneer", name) + return None + cfg = VersioneerConfig() + cfg.VCS = VCS + cfg.style = get(parser, "style") or "" + cfg.versionfile_source = get(parser, "versionfile_source") + cfg.versionfile_build = get(parser, "versionfile_build") + cfg.tag_prefix = get(parser, "tag_prefix") + cfg.parentdir_prefix = get(parser, "parentdir_prefix") + cfg.verbose = get(parser, "verbose") + return cfg + + +class NotThisMethod(Exception): + pass + +# these dictionaries contain VCS-specific tools +LONG_VERSION_PY = {} +HANDLERS = {} + + +def register_vcs_handler(vcs, method): # decorator + def decorate(f): + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate + + +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) + break + except EnvironmentError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %s" % dispcmd) + print(e) + return None + else: + if verbose: + print("unable to find command, tried %s" % (commands,)) + return None + stdout = p.communicate()[0].strip() + if sys.version_info[0] >= 3: + stdout = stdout.decode() + if p.returncode != 0: + if verbose: + print("unable to run %s (error)" % dispcmd) + return None + return stdout +LONG_VERSION_PY['git'] = ''' +# This file helps to compute a version number in source trees obtained from +# git-archive tarball (such as those provided by githubs download-from-tag +# feature). Distribution tarballs (built by setup.py sdist) and build +# directories (produced by setup.py build) will contain a much shorter file +# that just contains the computed version number. + +# This file is released into the public domain. Generated by +# versioneer-0.15 (https://github.com/warner/python-versioneer) + +import errno +import os +import re +import subprocess +import sys + + +def get_keywords(): + # these strings will be replaced by git during git-archive. + # setup.py/versioneer.py will grep for the variable names, so they must + # each be defined on a line of their own. _version.py will just call + # get_keywords(). + git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" + git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" + keywords = {"refnames": git_refnames, "full": git_full} + return keywords + + +class VersioneerConfig: + pass + + +def get_config(): + # these strings are filled in when 'setup.py versioneer' creates + # _version.py + cfg = VersioneerConfig() + cfg.VCS = "git" + cfg.style = "%(STYLE)s" + cfg.tag_prefix = "%(TAG_PREFIX)s" + cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s" + cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s" + cfg.verbose = False + return cfg + + +class NotThisMethod(Exception): + pass + + +LONG_VERSION_PY = {} +HANDLERS = {} + + +def register_vcs_handler(vcs, method): # decorator + def decorate(f): + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate + + +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) + break + except EnvironmentError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %%s" %% dispcmd) + print(e) + return None + else: + if verbose: + print("unable to find command, tried %%s" %% (commands,)) + return None + stdout = p.communicate()[0].strip() + if sys.version_info[0] >= 3: + stdout = stdout.decode() + if p.returncode != 0: + if verbose: + print("unable to run %%s (error)" %% dispcmd) + return None + return stdout + + +def versions_from_parentdir(parentdir_prefix, root, verbose): + # Source tarballs conventionally unpack into a directory that includes + # both the project name and a version string. + dirname = os.path.basename(root) + if not dirname.startswith(parentdir_prefix): + if verbose: + print("guessing rootdir is '%%s', but '%%s' doesn't start with " + "prefix '%%s'" %% (root, dirname, parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None} + + +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + # the code embedded in _version.py can just fetch the value of these + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} + try: + f = open(versionfile_abs, "r") + for line in f.readlines(): + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + f.close() + except EnvironmentError: + pass + return keywords + + +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + if not keywords: + raise NotThisMethod("no keywords at all, weird") + refnames = keywords["refnames"].strip() + if refnames.startswith("$Format"): + if verbose: + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") + refs = set([r.strip() for r in refnames.strip("()").split(",")]) + # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of + # just "foo-1.0". If we see a "tag: " prefix, prefer those. + TAG = "tag: " + tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + if not tags: + # Either we're using git < 1.8.3, or there really are no tags. We use + # a heuristic: assume all version tags have a digit. The old git %%d + # expansion behaves like git log --decorate=short and strips out the + # refs/heads/ and refs/tags/ prefixes that would let us distinguish + # between branches and tags. By ignoring refnames without digits, we + # filter out many common branch names like "release" and + # "stabilization", as well as "HEAD" and "master". + tags = set([r for r in refs if re.search(r'\d', r)]) + if verbose: + print("discarding '%%s', no digits" %% ",".join(refs-tags)) + if verbose: + print("likely tags: %%s" %% ",".join(sorted(tags))) + for ref in sorted(tags): + # sorting will prefer e.g. "2.0" over "2.0rc1" + if ref.startswith(tag_prefix): + r = ref[len(tag_prefix):] + if verbose: + print("picking %%s" %% r) + return {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None + } + # no suitable tags, so version is "0+unknown", but full hex is still there + if verbose: + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags"} + + +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + # this runs 'git' from the root of the source tree. This only gets called + # if the git-archive 'subst' keywords were *not* expanded, and + # _version.py hasn't already been rewritten with a short version string, + # meaning we're inside a checked out source tree. + + if not os.path.exists(os.path.join(root, ".git")): + if verbose: + print("no .git in %%s" %% root) + raise NotThisMethod("no .git directory") + + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] + # if there is a tag, this yields TAG-NUM-gHEX[-dirty] + # if there are no tags, this yields HEX[-dirty] (no NUM) + describe_out = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long"], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%%s'" + %% describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%%s' doesn't start with prefix '%%s'" + print(fmt %% (full_tag, tag_prefix)) + pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'" + %% (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + + else: + # HEX: no tags + pieces["closest-tag"] = None + count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits + + return pieces + + +def plus_or_dot(pieces): + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" + + +def render_pep440(pieces): + # now build up version string, with post-release "local version + # identifier". Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + # get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + # exceptions: + # 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + # TAG[.post.devDISTANCE] . No -dirty + + # exceptions: + # 1: no tags. 0.post.devDISTANCE + + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post.dev%%d" %% pieces["distance"] + else: + # exception #1 + rendered = "0.post.dev%%d" %% pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + # TAG[.postDISTANCE[.dev0]+gHEX] . The ".dev0" means dirty. Note that + # .dev0 sorts backwards (a dirty tree will appear "older" than the + # corresponding clean one), but you shouldn't be releasing software with + # -dirty anyways. + + # exceptions: + # 1: no tags. 0.postDISTANCE[.dev0] + + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%%s" %% pieces["short"] + else: + # exception #1 + rendered = "0.post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%%s" %% pieces["short"] + return rendered + + +def render_pep440_old(pieces): + # TAG[.postDISTANCE[.dev0]] . The ".dev0" means dirty. + + # exceptions: + # 1: no tags. 0.postDISTANCE[.dev0] + + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered + + +def render_git_describe(pieces): + # TAG[-DISTANCE-gHEX][-dirty], like 'git describe --tags --dirty + # --always' + + # exceptions: + # 1: no tags. HEX[-dirty] (note: no 'g' prefix) + + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces): + # TAG-DISTANCE-gHEX[-dirty], like 'git describe --tags --dirty + # --always -long'. The distance/hash is unconditional. + + # exceptions: + # 1: no tags. HEX[-dirty] (note: no 'g' prefix) + + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"]} + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%%s'" %% style) + + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None} + + +def get_versions(): + # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have + # __file__, we can work backwards from there to the root. Some + # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which + # case we can only use expanded keywords. + + cfg = get_config() + verbose = cfg.verbose + + try: + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, + verbose) + except NotThisMethod: + pass + + try: + root = os.path.realpath(__file__) + # versionfile_source is the relative path from the top of the source + # tree (where the .git directory might live) to this file. Invert + # this to find the root from __file__. + for i in cfg.versionfile_source.split('/'): + root = os.path.dirname(root) + except NameError: + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree"} + + try: + pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) + return render(pieces, cfg.style) + except NotThisMethod: + pass + + try: + if cfg.parentdir_prefix: + return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + except NotThisMethod: + pass + + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to compute version"} +''' + + +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + # the code embedded in _version.py can just fetch the value of these + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} + try: + f = open(versionfile_abs, "r") + for line in f.readlines(): + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + f.close() + except EnvironmentError: + pass + return keywords + + +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + if not keywords: + raise NotThisMethod("no keywords at all, weird") + refnames = keywords["refnames"].strip() + if refnames.startswith("$Format"): + if verbose: + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") + refs = set([r.strip() for r in refnames.strip("()").split(",")]) + # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of + # just "foo-1.0". If we see a "tag: " prefix, prefer those. + TAG = "tag: " + tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + if not tags: + # Either we're using git < 1.8.3, or there really are no tags. We use + # a heuristic: assume all version tags have a digit. The old git %d + # expansion behaves like git log --decorate=short and strips out the + # refs/heads/ and refs/tags/ prefixes that would let us distinguish + # between branches and tags. By ignoring refnames without digits, we + # filter out many common branch names like "release" and + # "stabilization", as well as "HEAD" and "master". + tags = set([r for r in refs if re.search(r'\d', r)]) + if verbose: + print("discarding '%s', no digits" % ",".join(refs-tags)) + if verbose: + print("likely tags: %s" % ",".join(sorted(tags))) + for ref in sorted(tags): + # sorting will prefer e.g. "2.0" over "2.0rc1" + if ref.startswith(tag_prefix): + r = ref[len(tag_prefix):] + if verbose: + print("picking %s" % r) + return {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None + } + # no suitable tags, so version is "0+unknown", but full hex is still there + if verbose: + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags"} + + +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + # this runs 'git' from the root of the source tree. This only gets called + # if the git-archive 'subst' keywords were *not* expanded, and + # _version.py hasn't already been rewritten with a short version string, + # meaning we're inside a checked out source tree. + + if not os.path.exists(os.path.join(root, ".git")): + if verbose: + print("no .git in %s" % root) + raise NotThisMethod("no .git directory") + + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] + # if there is a tag, this yields TAG-NUM-gHEX[-dirty] + # if there are no tags, this yields HEX[-dirty] (no NUM) + describe_out = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long"], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%s'" + % describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%s' doesn't start with prefix '%s'" + print(fmt % (full_tag, tag_prefix)) + pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" + % (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + + else: + # HEX: no tags + pieces["closest-tag"] = None + count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits + + return pieces + + +def do_vcs_install(manifest_in, versionfile_source, ipy): + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] + files = [manifest_in, versionfile_source] + if ipy: + files.append(ipy) + try: + me = __file__ + if me.endswith(".pyc") or me.endswith(".pyo"): + me = os.path.splitext(me)[0] + ".py" + versioneer_file = os.path.relpath(me) + except NameError: + versioneer_file = "versioneer.py" + files.append(versioneer_file) + present = False + try: + f = open(".gitattributes", "r") + for line in f.readlines(): + if line.strip().startswith(versionfile_source): + if "export-subst" in line.strip().split()[1:]: + present = True + f.close() + except EnvironmentError: + pass + if not present: + f = open(".gitattributes", "a+") + f.write("%s export-subst\n" % versionfile_source) + f.close() + files.append(".gitattributes") + run_command(GITS, ["add", "--"] + files) + + +def versions_from_parentdir(parentdir_prefix, root, verbose): + # Source tarballs conventionally unpack into a directory that includes + # both the project name and a version string. + dirname = os.path.basename(root) + if not dirname.startswith(parentdir_prefix): + if verbose: + print("guessing rootdir is '%s', but '%s' doesn't start with " + "prefix '%s'" % (root, dirname, parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None} + +SHORT_VERSION_PY = """ +# This file was generated by 'versioneer.py' (0.15) from +# revision-control system data, or from the parent directory name of an +# unpacked source archive. Distribution tarballs contain a pre-generated copy +# of this file. + +import json +import sys + +version_json = ''' +%s +''' # END VERSION_JSON + + +def get_versions(): + return json.loads(version_json) +""" + + +def versions_from_file(filename): + try: + with open(filename) as f: + contents = f.read() + except EnvironmentError: + raise NotThisMethod("unable to read _version.py") + mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", + contents, re.M | re.S) + if not mo: + raise NotThisMethod("no version_json in _version.py") + return json.loads(mo.group(1)) + + +def write_to_version_file(filename, versions): + os.unlink(filename) + contents = json.dumps(versions, sort_keys=True, + indent=1, separators=(",", ": ")) + with open(filename, "w") as f: + f.write(SHORT_VERSION_PY % contents) + + print("set %s to '%s'" % (filename, versions["version"])) + + +def plus_or_dot(pieces): + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" + + +def render_pep440(pieces): + # now build up version string, with post-release "local version + # identifier". Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + # get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + # exceptions: + # 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + # TAG[.post.devDISTANCE] . No -dirty + + # exceptions: + # 1: no tags. 0.post.devDISTANCE + + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post.dev%d" % pieces["distance"] + else: + # exception #1 + rendered = "0.post.dev%d" % pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + # TAG[.postDISTANCE[.dev0]+gHEX] . The ".dev0" means dirty. Note that + # .dev0 sorts backwards (a dirty tree will appear "older" than the + # corresponding clean one), but you shouldn't be releasing software with + # -dirty anyways. + + # exceptions: + # 1: no tags. 0.postDISTANCE[.dev0] + + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + return rendered + + +def render_pep440_old(pieces): + # TAG[.postDISTANCE[.dev0]] . The ".dev0" means dirty. + + # exceptions: + # 1: no tags. 0.postDISTANCE[.dev0] + + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered + + +def render_git_describe(pieces): + # TAG[-DISTANCE-gHEX][-dirty], like 'git describe --tags --dirty + # --always' + + # exceptions: + # 1: no tags. HEX[-dirty] (note: no 'g' prefix) + + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces): + # TAG-DISTANCE-gHEX[-dirty], like 'git describe --tags --dirty + # --always -long'. The distance/hash is unconditional. + + # exceptions: + # 1: no tags. HEX[-dirty] (note: no 'g' prefix) + + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"]} + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%s'" % style) + + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None} + + +class VersioneerBadRootError(Exception): + pass + + +def get_versions(verbose=False): + # returns dict with two keys: 'version' and 'full' + + if "versioneer" in sys.modules: + # see the discussion in cmdclass.py:get_cmdclass() + del sys.modules["versioneer"] + + root = get_root() + cfg = get_config_from_root(root) + + assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg" + handlers = HANDLERS.get(cfg.VCS) + assert handlers, "unrecognized VCS '%s'" % cfg.VCS + verbose = verbose or cfg.verbose + assert cfg.versionfile_source is not None, \ + "please set versioneer.versionfile_source" + assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" + + versionfile_abs = os.path.join(root, cfg.versionfile_source) + + # extract version from first of: _version.py, VCS command (e.g. 'git + # describe'), parentdir. This is meant to work for developers using a + # source checkout, for users of a tarball created by 'setup.py sdist', + # and for users of a tarball/zipball created by 'git archive' or github's + # download-from-tag feature or the equivalent in other VCSes. + + get_keywords_f = handlers.get("get_keywords") + from_keywords_f = handlers.get("keywords") + if get_keywords_f and from_keywords_f: + try: + keywords = get_keywords_f(versionfile_abs) + ver = from_keywords_f(keywords, cfg.tag_prefix, verbose) + if verbose: + print("got version from expanded keyword %s" % ver) + return ver + except NotThisMethod: + pass + + try: + ver = versions_from_file(versionfile_abs) + if verbose: + print("got version from file %s %s" % (versionfile_abs, ver)) + return ver + except NotThisMethod: + pass + + from_vcs_f = handlers.get("pieces_from_vcs") + if from_vcs_f: + try: + pieces = from_vcs_f(cfg.tag_prefix, root, verbose) + ver = render(pieces, cfg.style) + if verbose: + print("got version from VCS %s" % ver) + return ver + except NotThisMethod: + pass + + try: + if cfg.parentdir_prefix: + ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + if verbose: + print("got version from parentdir %s" % ver) + return ver + except NotThisMethod: + pass + + if verbose: + print("unable to compute version") + + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, "error": "unable to compute version"} + + +def get_version(): + return get_versions()["version"] + + +def get_cmdclass(): + if "versioneer" in sys.modules: + del sys.modules["versioneer"] + # this fixes the "python setup.py develop" case (also 'install' and + # 'easy_install .'), in which subdependencies of the main project are + # built (using setup.py bdist_egg) in the same python process. Assume + # a main project A and a dependency B, which use different versions + # of Versioneer. A's setup.py imports A's Versioneer, leaving it in + # sys.modules by the time B's setup.py is executed, causing B to run + # with the wrong versioneer. Setuptools wraps the sub-dep builds in a + # sandbox that restores sys.modules to it's pre-build state, so the + # parent is protected against the child's "import versioneer". By + # removing ourselves from sys.modules here, before the child build + # happens, we protect the child from the parent's versioneer too. + # Also see https://github.com/warner/python-versioneer/issues/52 + + cmds = {} + + # we add "version" to both distutils and setuptools + from distutils.core import Command + + class cmd_version(Command): + description = "report generated version string" + user_options = [] + boolean_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + vers = get_versions(verbose=True) + print("Version: %s" % vers["version"]) + print(" full-revisionid: %s" % vers.get("full-revisionid")) + print(" dirty: %s" % vers.get("dirty")) + if vers["error"]: + print(" error: %s" % vers["error"]) + cmds["version"] = cmd_version + + # we override "build_py" in both distutils and setuptools + # + # most invocation pathways end up running build_py: + # distutils/build -> build_py + # distutils/install -> distutils/build ->.. + # setuptools/bdist_wheel -> distutils/install ->.. + # setuptools/bdist_egg -> distutils/install_lib -> build_py + # setuptools/install -> bdist_egg ->.. + # setuptools/develop -> ? + + from distutils.command.build_py import build_py as _build_py + + class cmd_build_py(_build_py): + def run(self): + root = get_root() + cfg = get_config_from_root(root) + versions = get_versions() + _build_py.run(self) + # now locate _version.py in the new build/ directory and replace + # it with an updated value + if cfg.versionfile_build: + target_versionfile = os.path.join(self.build_lib, + cfg.versionfile_build) + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, versions) + cmds["build_py"] = cmd_build_py + + if "cx_Freeze" in sys.modules: # cx_freeze enabled? + from cx_Freeze.dist import build_exe as _build_exe + + class cmd_build_exe(_build_exe): + def run(self): + root = get_root() + cfg = get_config_from_root(root) + versions = get_versions() + target_versionfile = cfg.versionfile_source + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, versions) + + _build_exe.run(self) + os.unlink(target_versionfile) + with open(cfg.versionfile_source, "w") as f: + LONG = LONG_VERSION_PY[cfg.VCS] + f.write(LONG % + {"DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + }) + cmds["build_exe"] = cmd_build_exe + del cmds["build_py"] + + # we override different "sdist" commands for both environments + if "setuptools" in sys.modules: + from setuptools.command.sdist import sdist as _sdist + else: + from distutils.command.sdist import sdist as _sdist + + class cmd_sdist(_sdist): + def run(self): + versions = get_versions() + self._versioneer_generated_versions = versions + # unless we update this, the command will keep using the old + # version + self.distribution.metadata.version = versions["version"] + return _sdist.run(self) + + def make_release_tree(self, base_dir, files): + root = get_root() + cfg = get_config_from_root(root) + _sdist.make_release_tree(self, base_dir, files) + # now locate _version.py in the new base_dir directory + # (remembering that it may be a hardlink) and replace it with an + # updated value + target_versionfile = os.path.join(base_dir, cfg.versionfile_source) + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, + self._versioneer_generated_versions) + cmds["sdist"] = cmd_sdist + + return cmds + + +CONFIG_ERROR = """ +setup.cfg is missing the necessary Versioneer configuration. You need +a section like: + + [versioneer] + VCS = git + style = pep440 + versionfile_source = src/myproject/_version.py + versionfile_build = myproject/_version.py + tag_prefix = "" + parentdir_prefix = myproject- + +You will also need to edit your setup.py to use the results: + + import versioneer + setup(version=versioneer.get_version(), + cmdclass=versioneer.get_cmdclass(), ...) + +Please read the docstring in ./versioneer.py for configuration instructions, +edit setup.cfg, and re-run the installer or 'python versioneer.py setup'. +""" + +SAMPLE_CONFIG = """ +# See the docstring in versioneer.py for instructions. Note that you must +# re-run 'versioneer.py setup' after changing this section, and commit the +# resulting files. + +[versioneer] +#VCS = git +#style = pep440 +#versionfile_source = +#versionfile_build = +#tag_prefix = +#parentdir_prefix = + +""" + +INIT_PY_SNIPPET = """ +from ._version import get_versions +__version__ = get_versions()['version'] +del get_versions +""" + + +def do_setup(): + root = get_root() + try: + cfg = get_config_from_root(root) + except (EnvironmentError, configparser.NoSectionError, + configparser.NoOptionError) as e: + if isinstance(e, (EnvironmentError, configparser.NoSectionError)): + print("Adding sample versioneer config to setup.cfg", + file=sys.stderr) + with open(os.path.join(root, "setup.cfg"), "a") as f: + f.write(SAMPLE_CONFIG) + print(CONFIG_ERROR, file=sys.stderr) + return 1 + + print(" creating %s" % cfg.versionfile_source) + with open(cfg.versionfile_source, "w") as f: + LONG = LONG_VERSION_PY[cfg.VCS] + f.write(LONG % {"DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + }) + + ipy = os.path.join(os.path.dirname(cfg.versionfile_source), + "__init__.py") + if os.path.exists(ipy): + try: + with open(ipy, "r") as f: + old = f.read() + except EnvironmentError: + old = "" + if INIT_PY_SNIPPET not in old: + print(" appending to %s" % ipy) + with open(ipy, "a") as f: + f.write(INIT_PY_SNIPPET) + else: + print(" %s unmodified" % ipy) + else: + print(" %s doesn't exist, ok" % ipy) + ipy = None + + # Make sure both the top-level "versioneer.py" and versionfile_source + # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so + # they'll be copied into source distributions. Pip won't be able to + # install the package without this. + manifest_in = os.path.join(root, "MANIFEST.in") + simple_includes = set() + try: + with open(manifest_in, "r") as f: + for line in f: + if line.startswith("include "): + for include in line.split()[1:]: + simple_includes.add(include) + except EnvironmentError: + pass + # That doesn't cover everything MANIFEST.in can do + # (http://docs.python.org/2/distutils/sourcedist.html#commands), so + # it might give some false negatives. Appending redundant 'include' + # lines is safe, though. + if "versioneer.py" not in simple_includes: + print(" appending 'versioneer.py' to MANIFEST.in") + with open(manifest_in, "a") as f: + f.write("include versioneer.py\n") + else: + print(" 'versioneer.py' already in MANIFEST.in") + if cfg.versionfile_source not in simple_includes: + print(" appending versionfile_source ('%s') to MANIFEST.in" % + cfg.versionfile_source) + with open(manifest_in, "a") as f: + f.write("include %s\n" % cfg.versionfile_source) + else: + print(" versionfile_source already in MANIFEST.in") + + # Make VCS-specific changes. For git, this means creating/changing + # .gitattributes to mark _version.py for export-time keyword + # substitution. + do_vcs_install(manifest_in, cfg.versionfile_source, ipy) + return 0 + + +def scan_setup_py(): + found = set() + setters = False + errors = 0 + with open("setup.py", "r") as f: + for line in f.readlines(): + if "import versioneer" in line: + found.add("import") + if "versioneer.get_cmdclass()" in line: + found.add("cmdclass") + if "versioneer.get_version()" in line: + found.add("get_version") + if "versioneer.VCS" in line: + setters = True + if "versioneer.versionfile_source" in line: + setters = True + if len(found) != 3: + print("") + print("Your setup.py appears to be missing some important items") + print("(but I might be wrong). Please make sure it has something") + print("roughly like the following:") + print("") + print(" import versioneer") + print(" setup( version=versioneer.get_version(),") + print(" cmdclass=versioneer.get_cmdclass(), ...)") + print("") + errors += 1 + if setters: + print("You should remove lines like 'versioneer.VCS = ' and") + print("'versioneer.versionfile_source = ' . This configuration") + print("now lives in setup.cfg, and should be removed from setup.py") + print("") + errors += 1 + return errors + +if __name__ == "__main__": + cmd = sys.argv[1] + if cmd == "setup": + errors = do_setup() + errors += scan_setup_py() + if errors: + sys.exit(1) From a87686787271bcf5ae276d212fe8c715dd0572ff Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 8 Feb 2017 08:31:10 -0500 Subject: [PATCH 004/519] initial version from pandasv 0.19.0-419-gfe246cc --- packages/pandas-gbq/.gitignore | 3 + packages/pandas-gbq/pandas_gbq/__init__.py | 5 +- packages/pandas-gbq/pandas_gbq/gbq.py | 1158 ++++++++++++++++ .../pandas-gbq/pandas_gbq/tests/__init__.py | 4 - .../pandas-gbq/pandas_gbq/tests/test_gbq.py | 1230 +++++++++++++++++ 5 files changed, 2392 insertions(+), 8 deletions(-) create mode 100644 packages/pandas-gbq/pandas_gbq/gbq.py create mode 100644 packages/pandas-gbq/pandas_gbq/tests/test_gbq.py diff --git a/packages/pandas-gbq/.gitignore b/packages/pandas-gbq/.gitignore index eac15860886d..21b187c255ab 100644 --- a/packages/pandas-gbq/.gitignore +++ b/packages/pandas-gbq/.gitignore @@ -65,3 +65,6 @@ dist ehthumbs.db Icon? Thumbs.db + +# caches # +.cache diff --git a/packages/pandas-gbq/pandas_gbq/__init__.py b/packages/pandas-gbq/pandas_gbq/__init__.py index 40c60425991c..ad1658ec0c09 100644 --- a/packages/pandas-gbq/pandas_gbq/__init__.py +++ b/packages/pandas-gbq/pandas_gbq/__init__.py @@ -1,4 +1 @@ -__version__ = version = '0.3.0.post' - -from .data import (get_components_yahoo, get_data_famafrench, get_data_google, get_data_yahoo, get_data_enigma, # noqa - get_data_yahoo_actions, get_quote_google, get_quote_yahoo, DataReader, Options) # noqa +from .gbq import to_gbq, read_gbq # noqa diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py new file mode 100644 index 000000000000..966f53e9d75e --- /dev/null +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -0,0 +1,1158 @@ +import warnings +from datetime import datetime +import json +import logging +from time import sleep +import uuid +import time +import sys + +import numpy as np + +from distutils.version import StrictVersion +from pandas import compat +from pandas.core.api import DataFrame +from pandas.tools.merge import concat +from pandas.core.common import PandasError +from pandas.compat import lzip, bytes_to_str + + +def _check_google_client_version(): + + try: + import pkg_resources + + except ImportError: + raise ImportError('Could not import pkg_resources (setuptools).') + + if compat.PY3: + google_api_minimum_version = '1.4.1' + else: + google_api_minimum_version = '1.2.0' + + _GOOGLE_API_CLIENT_VERSION = pkg_resources.get_distribution( + 'google-api-python-client').version + + if (StrictVersion(_GOOGLE_API_CLIENT_VERSION) < + StrictVersion(google_api_minimum_version)): + raise ImportError("pandas requires google-api-python-client >= {0} " + "for Google BigQuery support, " + "current version {1}" + .format(google_api_minimum_version, + _GOOGLE_API_CLIENT_VERSION)) + + +def _test_google_api_imports(): + + try: + import httplib2 # noqa + try: + from googleapiclient.discovery import build # noqa + from googleapiclient.errors import HttpError # noqa + except: + from apiclient.discovery import build # noqa + from apiclient.errors import HttpError # noqa + from oauth2client.client import AccessTokenRefreshError # noqa + from oauth2client.client import OAuth2WebServerFlow # noqa + from oauth2client.file import Storage # noqa + from oauth2client.tools import run_flow, argparser # noqa + except ImportError as e: + raise ImportError("Missing module required for Google BigQuery " + "support: {0}".format(str(e))) + +logger = logging.getLogger('pandas.io.gbq') +logger.setLevel(logging.ERROR) + + +class InvalidPrivateKeyFormat(PandasError, ValueError): + """ + Raised when provided private key has invalid format. + """ + pass + + +class AccessDenied(PandasError, ValueError): + """ + Raised when invalid credentials are provided, or tokens have expired. + """ + pass + + +class DatasetCreationError(PandasError, ValueError): + """ + Raised when the create dataset method fails + """ + pass + + +class GenericGBQException(PandasError, ValueError): + """ + Raised when an unrecognized Google API Error occurs. + """ + pass + + +class InvalidColumnOrder(PandasError, ValueError): + """ + Raised when the provided column order for output + results DataFrame does not match the schema + returned by BigQuery. + """ + pass + + +class InvalidPageToken(PandasError, ValueError): + """ + Raised when Google BigQuery fails to return, + or returns a duplicate page token. + """ + pass + + +class InvalidSchema(PandasError, ValueError): + """ + Raised when the provided DataFrame does + not match the schema of the destination + table in BigQuery. + """ + pass + + +class NotFoundException(PandasError, ValueError): + """ + Raised when the project_id, table or dataset provided in the query could + not be found. + """ + pass + + +class StreamingInsertError(PandasError, ValueError): + """ + Raised when BigQuery reports a streaming insert error. + For more information see `Streaming Data Into BigQuery + `__ + """ + + +class TableCreationError(PandasError, ValueError): + """ + Raised when the create table method fails + """ + pass + + +class GbqConnector(object): + scope = 'https://www.googleapis.com/auth/bigquery' + + def __init__(self, project_id, reauth=False, verbose=False, + private_key=None, dialect='legacy'): + _check_google_client_version() + _test_google_api_imports() + self.project_id = project_id + self.reauth = reauth + self.verbose = verbose + self.private_key = private_key + self.dialect = dialect + self.credentials = self.get_credentials() + self.service = self.get_service() + + def get_credentials(self): + if self.private_key: + return self.get_service_account_credentials() + else: + # Try to retrieve Application Default Credentials + credentials = self.get_application_default_credentials() + if not credentials: + credentials = self.get_user_account_credentials() + return credentials + + def get_application_default_credentials(self): + """ + This method tries to retrieve the "default application credentials". + This could be useful for running code on Google Cloud Platform. + + .. versionadded:: 0.19.0 + + Parameters + ---------- + None + + Returns + ------- + - GoogleCredentials, + If the default application credentials can be retrieved + from the environment. The retrieved credentials should also + have access to the project (self.project_id) on BigQuery. + - OR None, + If default application credentials can not be retrieved + from the environment. Or, the retrieved credentials do not + have access to the project (self.project_id) on BigQuery. + """ + import httplib2 + try: + from googleapiclient.discovery import build + except ImportError: + from apiclient.discovery import build + try: + from oauth2client.client import GoogleCredentials + except ImportError: + return None + + try: + credentials = GoogleCredentials.get_application_default() + except: + return None + + http = httplib2.Http() + try: + http = credentials.authorize(http) + bigquery_service = build('bigquery', 'v2', http=http) + # Check if the application has rights to the BigQuery project + jobs = bigquery_service.jobs() + job_data = {'configuration': {'query': {'query': 'SELECT 1'}}} + jobs.insert(projectId=self.project_id, body=job_data).execute() + return credentials + except: + return None + + def get_user_account_credentials(self): + from oauth2client.client import OAuth2WebServerFlow + from oauth2client.file import Storage + from oauth2client.tools import run_flow, argparser + + flow = OAuth2WebServerFlow( + client_id=('495642085510-k0tmvj2m941jhre2nbqka17vqpjfddtd' + '.apps.googleusercontent.com'), + client_secret='kOc9wMptUtxkcIFbtZCcrEAc', + scope=self.scope, + redirect_uri='urn:ietf:wg:oauth:2.0:oob') + + storage = Storage('bigquery_credentials.dat') + credentials = storage.get() + + if credentials is None or credentials.invalid or self.reauth: + credentials = run_flow(flow, storage, argparser.parse_args([])) + + return credentials + + def get_service_account_credentials(self): + # Bug fix for https://github.com/pandas-dev/pandas/issues/12572 + # We need to know that a supported version of oauth2client is installed + # Test that either of the following is installed: + # - SignedJwtAssertionCredentials from oauth2client.client + # - ServiceAccountCredentials from oauth2client.service_account + # SignedJwtAssertionCredentials is available in oauthclient < 2.0.0 + # ServiceAccountCredentials is available in oauthclient >= 2.0.0 + oauth2client_v1 = True + oauth2client_v2 = True + + try: + from oauth2client.client import SignedJwtAssertionCredentials + except ImportError: + oauth2client_v1 = False + + try: + from oauth2client.service_account import ServiceAccountCredentials + except ImportError: + oauth2client_v2 = False + + if not oauth2client_v1 and not oauth2client_v2: + raise ImportError("Missing oauth2client required for BigQuery " + "service account support") + + from os.path import isfile + + try: + if isfile(self.private_key): + with open(self.private_key) as f: + json_key = json.loads(f.read()) + else: + # ugly hack: 'private_key' field has new lines inside, + # they break json parser, but we need to preserve them + json_key = json.loads(self.private_key.replace('\n', ' ')) + json_key['private_key'] = json_key['private_key'].replace( + ' ', '\n') + + if compat.PY3: + json_key['private_key'] = bytes( + json_key['private_key'], 'UTF-8') + + if oauth2client_v1: + return SignedJwtAssertionCredentials( + json_key['client_email'], + json_key['private_key'], + self.scope, + ) + else: + return ServiceAccountCredentials.from_json_keyfile_dict( + json_key, + self.scope) + except (KeyError, ValueError, TypeError, AttributeError): + raise InvalidPrivateKeyFormat( + "Private key is missing or invalid. It should be service " + "account private key JSON (file path or string contents) " + "with at least two keys: 'client_email' and 'private_key'. " + "Can be obtained from: https://console.developers.google." + "com/permissions/serviceaccounts") + + def _print(self, msg, end='\n'): + if self.verbose: + sys.stdout.write(msg + end) + sys.stdout.flush() + + def _start_timer(self): + self.start = time.time() + + def get_elapsed_seconds(self): + return round(time.time() - self.start, 2) + + def print_elapsed_seconds(self, prefix='Elapsed', postfix='s.', + overlong=7): + sec = self.get_elapsed_seconds() + if sec > overlong: + self._print('{} {} {}'.format(prefix, sec, postfix)) + + # http://stackoverflow.com/questions/1094841/reusable-library-to-get-human-readable-version-of-file-size + @staticmethod + def sizeof_fmt(num, suffix='b'): + fmt = "%3.1f %s%s" + for unit in ['', 'k', 'M', 'G', 'T', 'P', 'E', 'Z']: + if abs(num) < 1024.0: + return fmt % (num, unit, suffix) + num /= 1024.0 + return fmt % (num, 'Y', suffix) + + def get_service(self): + import httplib2 + try: + from googleapiclient.discovery import build + except: + from apiclient.discovery import build + + http = httplib2.Http() + http = self.credentials.authorize(http) + bigquery_service = build('bigquery', 'v2', http=http) + + return bigquery_service + + @staticmethod + def process_http_error(ex): + # See `BigQuery Troubleshooting Errors + # `__ + + status = json.loads(bytes_to_str(ex.content))['error'] + errors = status.get('errors', None) + + if errors: + for error in errors: + reason = error['reason'] + message = error['message'] + + raise GenericGBQException( + "Reason: {0}, Message: {1}".format(reason, message)) + + raise GenericGBQException(errors) + + def process_insert_errors(self, insert_errors): + for insert_error in insert_errors: + row = insert_error['index'] + errors = insert_error.get('errors', None) + for error in errors: + reason = error['reason'] + message = error['message'] + location = error['location'] + error_message = ('Error at Row: {0}, Reason: {1}, ' + 'Location: {2}, Message: {3}' + .format(row, reason, location, message)) + + # Report all error messages if verbose is set + if self.verbose: + self._print(error_message) + else: + raise StreamingInsertError(error_message + + '\nEnable verbose logging to ' + 'see all errors') + + raise StreamingInsertError + + def run_query(self, query, **kwargs): + try: + from googleapiclient.errors import HttpError + except: + from apiclient.errors import HttpError + from oauth2client.client import AccessTokenRefreshError + + _check_google_client_version() + + job_collection = self.service.jobs() + + job_config = { + 'query': { + 'query': query, + 'useLegacySql': self.dialect == 'legacy' + # 'allowLargeResults', 'createDisposition', + # 'preserveNulls', destinationTable, useQueryCache + } + } + config = kwargs.get('configuration') + if config is not None: + if len(config) != 1: + raise ValueError("Only one job type must be specified, but " + "given {}".format(','.join(config.keys()))) + if 'query' in config: + if 'query' in config['query'] and query is not None: + raise ValueError("Query statement can't be specified " + "inside config while it is specified " + "as parameter") + + job_config['query'].update(config['query']) + else: + raise ValueError("Only 'query' job type is supported") + + job_data = { + 'configuration': job_config + } + + self._start_timer() + try: + self._print('Requesting query... ', end="") + query_reply = job_collection.insert( + projectId=self.project_id, body=job_data).execute() + self._print('ok.\nQuery running...') + except (AccessTokenRefreshError, ValueError): + if self.private_key: + raise AccessDenied( + "The service account credentials are not valid") + else: + raise AccessDenied( + "The credentials have been revoked or expired, " + "please re-run the application to re-authorize") + except HttpError as ex: + self.process_http_error(ex) + + job_reference = query_reply['jobReference'] + + while not query_reply.get('jobComplete', False): + self.print_elapsed_seconds(' Elapsed', 's. Waiting...') + try: + query_reply = job_collection.getQueryResults( + projectId=job_reference['projectId'], + jobId=job_reference['jobId']).execute() + except HttpError as ex: + self.process_http_error(ex) + + if self.verbose: + if query_reply['cacheHit']: + self._print('Query done.\nCache hit.\n') + else: + bytes_processed = int(query_reply.get( + 'totalBytesProcessed', '0')) + self._print('Query done.\nProcessed: {}\n'.format( + self.sizeof_fmt(bytes_processed))) + + self._print('Retrieving results...') + + total_rows = int(query_reply['totalRows']) + result_pages = list() + seen_page_tokens = list() + current_row = 0 + # Only read schema on first page + schema = query_reply['schema'] + + # Loop through each page of data + while 'rows' in query_reply and current_row < total_rows: + page = query_reply['rows'] + result_pages.append(page) + current_row += len(page) + + self.print_elapsed_seconds( + ' Got page: {}; {}% done. Elapsed'.format( + len(result_pages), + round(100.0 * current_row / total_rows))) + + if current_row == total_rows: + break + + page_token = query_reply.get('pageToken', None) + + if not page_token and current_row < total_rows: + raise InvalidPageToken("Required pageToken was missing. " + "Received {0} of {1} rows" + .format(current_row, total_rows)) + + elif page_token in seen_page_tokens: + raise InvalidPageToken("A duplicate pageToken was returned") + + seen_page_tokens.append(page_token) + + try: + query_reply = job_collection.getQueryResults( + projectId=job_reference['projectId'], + jobId=job_reference['jobId'], + pageToken=page_token).execute() + except HttpError as ex: + self.process_http_error(ex) + + if current_row < total_rows: + raise InvalidPageToken() + + # print basic query stats + self._print('Got {} rows.\n'.format(total_rows)) + + return schema, result_pages + + def load_data(self, dataframe, dataset_id, table_id, chunksize): + try: + from googleapiclient.errors import HttpError + except: + from apiclient.errors import HttpError + + job_id = uuid.uuid4().hex + rows = [] + remaining_rows = len(dataframe) + + total_rows = remaining_rows + self._print("\n\n") + + for index, row in dataframe.reset_index(drop=True).iterrows(): + row_dict = dict() + row_dict['json'] = json.loads(row.to_json(force_ascii=False, + date_unit='s', + date_format='iso')) + row_dict['insertId'] = job_id + str(index) + rows.append(row_dict) + remaining_rows -= 1 + + if (len(rows) % chunksize == 0) or (remaining_rows == 0): + self._print("\rStreaming Insert is {0}% Complete".format( + ((total_rows - remaining_rows) * 100) / total_rows)) + + body = {'rows': rows} + + try: + response = self.service.tabledata().insertAll( + projectId=self.project_id, + datasetId=dataset_id, + tableId=table_id, + body=body).execute() + except HttpError as ex: + self.process_http_error(ex) + + # For streaming inserts, even if you receive a success HTTP + # response code, you'll need to check the insertErrors property + # of the response to determine if the row insertions were + # successful, because it's possible that BigQuery was only + # partially successful at inserting the rows. See the `Success + # HTTP Response Codes + # `__ + # section + + insert_errors = response.get('insertErrors', None) + if insert_errors: + self.process_insert_errors(insert_errors) + + sleep(1) # Maintains the inserts "per second" rate per API + rows = [] + + self._print("\n") + + def verify_schema(self, dataset_id, table_id, schema): + try: + from googleapiclient.errors import HttpError + except: + from apiclient.errors import HttpError + + try: + remote_schema = self.service.tables().get( + projectId=self.project_id, + datasetId=dataset_id, + tableId=table_id).execute()['schema'] + + fields_remote = set([json.dumps(field_remote) + for field_remote in remote_schema['fields']]) + fields_local = set(json.dumps(field_local) + for field_local in schema['fields']) + + return fields_remote == fields_local + except HttpError as ex: + self.process_http_error(ex) + + def delete_and_recreate_table(self, dataset_id, table_id, table_schema): + delay = 0 + + # Changes to table schema may take up to 2 minutes as of May 2015 See + # `Issue 191 + # `__ + # Compare previous schema with new schema to determine if there should + # be a 120 second delay + + if not self.verify_schema(dataset_id, table_id, table_schema): + self._print('The existing table has a different schema. Please ' + 'wait 2 minutes. See Google BigQuery issue #191') + delay = 120 + + table = _Table(self.project_id, dataset_id, + private_key=self.private_key) + table.delete(table_id) + table.create(table_id, table_schema) + sleep(delay) + + +def _parse_data(schema, rows): + # see: + # http://pandas.pydata.org/pandas-docs/dev/missing_data.html + # #missing-data-casting-rules-and-indexing + dtype_map = {'INTEGER': np.dtype(float), + 'FLOAT': np.dtype(float), + # This seems to be buggy without nanosecond indicator + 'TIMESTAMP': 'M8[ns]'} + + fields = schema['fields'] + col_types = [field['type'] for field in fields] + col_names = [str(field['name']) for field in fields] + col_dtypes = [dtype_map.get(field['type'], object) for field in fields] + page_array = np.zeros((len(rows),), + dtype=lzip(col_names, col_dtypes)) + + for row_num, raw_row in enumerate(rows): + entries = raw_row.get('f', []) + for col_num, field_type in enumerate(col_types): + field_value = _parse_entry(entries[col_num].get('v', ''), + field_type) + page_array[row_num][col_num] = field_value + + return DataFrame(page_array, columns=col_names) + + +def _parse_entry(field_value, field_type): + if field_value is None or field_value == 'null': + return None + if field_type == 'INTEGER' or field_type == 'FLOAT': + return float(field_value) + elif field_type == 'TIMESTAMP': + timestamp = datetime.utcfromtimestamp(float(field_value)) + return np.datetime64(timestamp) + elif field_type == 'BOOLEAN': + return field_value == 'true' + return field_value + + +def read_gbq(query, project_id=None, index_col=None, col_order=None, + reauth=False, verbose=True, private_key=None, dialect='legacy', + **kwargs): + r"""Load data from Google BigQuery. + + THIS IS AN EXPERIMENTAL LIBRARY + + The main method a user calls to execute a Query in Google BigQuery + and read results into a pandas DataFrame. + + Google BigQuery API Client Library v2 for Python is used. + Documentation is available at + https://developers.google.com/api-client-library/python/apis/bigquery/v2 + + Authentication to the Google BigQuery service is via OAuth 2.0. + + - If "private_key" is not provided: + + By default "application default credentials" are used. + + .. versionadded:: 0.19.0 + + If default application credentials are not found or are restrictive, + user account credentials are used. In this case, you will be asked to + grant permissions for product name 'pandas GBQ'. + + - If "private_key" is provided: + + Service account credentials will be used to authenticate. + + Parameters + ---------- + query : str + SQL-Like Query to return data values + project_id : str + Google BigQuery Account project ID. + index_col : str (optional) + Name of result column to use for index in results DataFrame + col_order : list(str) (optional) + List of BigQuery column names in the desired order for results + DataFrame + reauth : boolean (default False) + Force Google BigQuery to reauthenticate the user. This is useful + if multiple accounts are used. + verbose : boolean (default True) + Verbose output + private_key : str (optional) + Service account private key in JSON format. Can be file path + or string contents. This is useful for remote server + authentication (eg. jupyter iPython notebook on remote host) + + .. versionadded:: 0.18.1 + + dialect : {'legacy', 'standard'}, default 'legacy' + 'legacy' : Use BigQuery's legacy SQL dialect. + 'standard' : Use BigQuery's standard SQL (beta), which is + compliant with the SQL 2011 standard. For more information + see `BigQuery SQL Reference + `__ + + .. versionadded:: 0.19.0 + + **kwargs : Arbitrary keyword arguments + configuration (dict): query config parameters for job processing. + For example: + + configuration = {'query': {'useQueryCache': False}} + + For more information see `BigQuery SQL Reference + ` + + .. versionadded:: 0.20.0 + + Returns + ------- + df: DataFrame + DataFrame representing results of query + + """ + + if not project_id: + raise TypeError("Missing required parameter: project_id") + + if dialect not in ('legacy', 'standard'): + raise ValueError("'{0}' is not valid for dialect".format(dialect)) + + connector = GbqConnector(project_id, reauth=reauth, verbose=verbose, + private_key=private_key, + dialect=dialect) + schema, pages = connector.run_query(query, **kwargs) + dataframe_list = [] + while len(pages) > 0: + page = pages.pop() + dataframe_list.append(_parse_data(schema, page)) + + if len(dataframe_list) > 0: + final_df = concat(dataframe_list, ignore_index=True) + else: + final_df = _parse_data(schema, []) + + # Reindex the DataFrame on the provided column + if index_col is not None: + if index_col in final_df.columns: + final_df.set_index(index_col, inplace=True) + else: + raise InvalidColumnOrder( + 'Index column "{0}" does not exist in DataFrame.' + .format(index_col) + ) + + # Change the order of columns in the DataFrame based on provided list + if col_order is not None: + if sorted(col_order) == sorted(final_df.columns): + final_df = final_df[col_order] + else: + raise InvalidColumnOrder( + 'Column order does not match this DataFrame.' + ) + + # Downcast floats to integers and objects to booleans + # if there are no NaN's. This is presently due to a + # limitation of numpy in handling missing data. + final_df._data = final_df._data.downcast(dtypes='infer') + + connector.print_elapsed_seconds( + 'Total time taken', + datetime.now().strftime('s.\nFinished at %Y-%m-%d %H:%M:%S.'), + 0 + ) + + return final_df + + +def to_gbq(dataframe, destination_table, project_id, chunksize=10000, + verbose=True, reauth=False, if_exists='fail', private_key=None): + """Write a DataFrame to a Google BigQuery table. + + THIS IS AN EXPERIMENTAL LIBRARY + + The main method a user calls to export pandas DataFrame contents to + Google BigQuery table. + + Google BigQuery API Client Library v2 for Python is used. + Documentation is available at + https://developers.google.com/api-client-library/python/apis/bigquery/v2 + + Authentication to the Google BigQuery service is via OAuth 2.0. + + - If "private_key" is not provided: + + By default "application default credentials" are used. + + .. versionadded:: 0.19.0 + + If default application credentials are not found or are restrictive, + user account credentials are used. In this case, you will be asked to + grant permissions for product name 'pandas GBQ'. + + - If "private_key" is provided: + + Service account credentials will be used to authenticate. + + Parameters + ---------- + dataframe : DataFrame + DataFrame to be written + destination_table : string + Name of table to be written, in the form 'dataset.tablename' + project_id : str + Google BigQuery Account project ID. + chunksize : int (default 10000) + Number of rows to be inserted in each chunk from the dataframe. + verbose : boolean (default True) + Show percentage complete + reauth : boolean (default False) + Force Google BigQuery to reauthenticate the user. This is useful + if multiple accounts are used. + if_exists : {'fail', 'replace', 'append'}, default 'fail' + 'fail': If table exists, do nothing. + 'replace': If table exists, drop it, recreate it, and insert data. + 'append': If table exists, insert data. Create if does not exist. + private_key : str (optional) + Service account private key in JSON format. Can be file path + or string contents. This is useful for remote server + authentication (eg. jupyter iPython notebook on remote host) + """ + + if if_exists not in ('fail', 'replace', 'append'): + raise ValueError("'{0}' is not valid for if_exists".format(if_exists)) + + if '.' not in destination_table: + raise NotFoundException( + "Invalid Table Name. Should be of the form 'datasetId.tableId' ") + + connector = GbqConnector(project_id, reauth=reauth, verbose=verbose, + private_key=private_key) + dataset_id, table_id = destination_table.rsplit('.', 1) + + table = _Table(project_id, dataset_id, reauth=reauth, + private_key=private_key) + + table_schema = _generate_bq_schema(dataframe) + + # If table exists, check if_exists parameter + if table.exists(table_id): + if if_exists == 'fail': + raise TableCreationError("Could not create the table because it " + "already exists. " + "Change the if_exists parameter to " + "append or replace data.") + elif if_exists == 'replace': + connector.delete_and_recreate_table( + dataset_id, table_id, table_schema) + elif if_exists == 'append': + if not connector.verify_schema(dataset_id, table_id, table_schema): + raise InvalidSchema("Please verify that the structure and " + "data types in the DataFrame match the " + "schema of the destination table.") + else: + table.create(table_id, table_schema) + + connector.load_data(dataframe, dataset_id, table_id, chunksize) + + +def generate_bq_schema(df, default_type='STRING'): + # deprecation TimeSeries, #11121 + warnings.warn("generate_bq_schema is deprecated and will be removed in " + "a future version", FutureWarning, stacklevel=2) + + return _generate_bq_schema(df, default_type=default_type) + + +def _generate_bq_schema(df, default_type='STRING'): + """ Given a passed df, generate the associated Google BigQuery schema. + + Parameters + ---------- + df : DataFrame + default_type : string + The default big query type in case the type of the column + does not exist in the schema. + """ + + type_mapping = { + 'i': 'INTEGER', + 'b': 'BOOLEAN', + 'f': 'FLOAT', + 'O': 'STRING', + 'S': 'STRING', + 'U': 'STRING', + 'M': 'TIMESTAMP' + } + + fields = [] + for column_name, dtype in df.dtypes.iteritems(): + fields.append({'name': column_name, + 'type': type_mapping.get(dtype.kind, default_type)}) + + return {'fields': fields} + + +class _Table(GbqConnector): + + def __init__(self, project_id, dataset_id, reauth=False, verbose=False, + private_key=None): + try: + from googleapiclient.errors import HttpError + except: + from apiclient.errors import HttpError + self.http_error = HttpError + self.dataset_id = dataset_id + super(_Table, self).__init__(project_id, reauth, verbose, private_key) + + def exists(self, table_id): + """ Check if a table exists in Google BigQuery + + .. versionadded:: 0.17.0 + + Parameters + ---------- + table : str + Name of table to be verified + + Returns + ------- + boolean + true if table exists, otherwise false + """ + + try: + self.service.tables().get( + projectId=self.project_id, + datasetId=self.dataset_id, + tableId=table_id).execute() + return True + except self.http_error as ex: + if ex.resp.status == 404: + return False + else: + self.process_http_error(ex) + + def create(self, table_id, schema): + """ Create a table in Google BigQuery given a table and schema + + .. versionadded:: 0.17.0 + + Parameters + ---------- + table : str + Name of table to be written + schema : str + Use the generate_bq_schema to generate your table schema from a + dataframe. + """ + + if self.exists(table_id): + raise TableCreationError( + "The table could not be created because it already exists") + + if not _Dataset(self.project_id, + private_key=self.private_key).exists(self.dataset_id): + _Dataset(self.project_id, + private_key=self.private_key).create(self.dataset_id) + + body = { + 'schema': schema, + 'tableReference': { + 'tableId': table_id, + 'projectId': self.project_id, + 'datasetId': self.dataset_id + } + } + + try: + self.service.tables().insert( + projectId=self.project_id, + datasetId=self.dataset_id, + body=body).execute() + except self.http_error as ex: + self.process_http_error(ex) + + def delete(self, table_id): + """ Delete a table in Google BigQuery + + .. versionadded:: 0.17.0 + + Parameters + ---------- + table : str + Name of table to be deleted + """ + + if not self.exists(table_id): + raise NotFoundException("Table does not exist") + + try: + self.service.tables().delete( + datasetId=self.dataset_id, + projectId=self.project_id, + tableId=table_id).execute() + except self.http_error as ex: + self.process_http_error(ex) + + +class _Dataset(GbqConnector): + + def __init__(self, project_id, reauth=False, verbose=False, + private_key=None): + try: + from googleapiclient.errors import HttpError + except: + from apiclient.errors import HttpError + self.http_error = HttpError + super(_Dataset, self).__init__(project_id, reauth, verbose, + private_key) + + def exists(self, dataset_id): + """ Check if a dataset exists in Google BigQuery + + .. versionadded:: 0.17.0 + + Parameters + ---------- + dataset_id : str + Name of dataset to be verified + + Returns + ------- + boolean + true if dataset exists, otherwise false + """ + + try: + self.service.datasets().get( + projectId=self.project_id, + datasetId=dataset_id).execute() + return True + except self.http_error as ex: + if ex.resp.status == 404: + return False + else: + self.process_http_error(ex) + + def datasets(self): + """ Return a list of datasets in Google BigQuery + + .. versionadded:: 0.17.0 + + Parameters + ---------- + None + + Returns + ------- + list + List of datasets under the specific project + """ + + try: + list_dataset_response = self.service.datasets().list( + projectId=self.project_id).execute().get('datasets', None) + + if not list_dataset_response: + return [] + + dataset_list = list() + + for row_num, raw_row in enumerate(list_dataset_response): + dataset_list.append(raw_row['datasetReference']['datasetId']) + + return dataset_list + except self.http_error as ex: + self.process_http_error(ex) + + def create(self, dataset_id): + """ Create a dataset in Google BigQuery + + .. versionadded:: 0.17.0 + + Parameters + ---------- + dataset : str + Name of dataset to be written + """ + + if self.exists(dataset_id): + raise DatasetCreationError( + "The dataset could not be created because it already exists") + + body = { + 'datasetReference': { + 'projectId': self.project_id, + 'datasetId': dataset_id + } + } + + try: + self.service.datasets().insert( + projectId=self.project_id, + body=body).execute() + except self.http_error as ex: + self.process_http_error(ex) + + def delete(self, dataset_id): + """ Delete a dataset in Google BigQuery + + .. versionadded:: 0.17.0 + + Parameters + ---------- + dataset : str + Name of dataset to be deleted + """ + + if not self.exists(dataset_id): + raise NotFoundException( + "Dataset {0} does not exist".format(dataset_id)) + + try: + self.service.datasets().delete( + datasetId=dataset_id, + projectId=self.project_id).execute() + + except self.http_error as ex: + self.process_http_error(ex) + + def tables(self, dataset_id): + """ List tables in the specific dataset in Google BigQuery + + .. versionadded:: 0.17.0 + + Parameters + ---------- + dataset : str + Name of dataset to list tables for + + Returns + ------- + list + List of tables under the specific dataset + """ + + try: + list_table_response = self.service.tables().list( + projectId=self.project_id, + datasetId=dataset_id).execute().get('tables', None) + + if not list_table_response: + return [] + + table_list = list() + + for row_num, raw_row in enumerate(list_table_response): + table_list.append(raw_row['tableReference']['tableId']) + + return table_list + except self.http_error as ex: + self.process_http_error(ex) diff --git a/packages/pandas-gbq/pandas_gbq/tests/__init__.py b/packages/pandas-gbq/pandas_gbq/tests/__init__.py index 40c60425991c..e69de29bb2d1 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/__init__.py +++ b/packages/pandas-gbq/pandas_gbq/tests/__init__.py @@ -1,4 +0,0 @@ -__version__ = version = '0.3.0.post' - -from .data import (get_components_yahoo, get_data_famafrench, get_data_google, get_data_yahoo, get_data_enigma, # noqa - get_data_yahoo_actions, get_quote_google, get_quote_yahoo, DataReader, Options) # noqa diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py new file mode 100644 index 000000000000..457e2d218cb3 --- /dev/null +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -0,0 +1,1230 @@ +import re +from datetime import datetime +import nose +import pytz +import platform +from time import sleep +import os +import logging + +import numpy as np + +from distutils.version import StrictVersion +from pandas import compat + +from pandas import NaT +from pandas.compat import u, range +from pandas.core.frame import DataFrame +import pandas.io.gbq as gbq +import pandas.util.testing as tm +from pandas.compat.numpy import np_datetime64_compat + +PROJECT_ID = None +PRIVATE_KEY_JSON_PATH = None +PRIVATE_KEY_JSON_CONTENTS = None + +if compat.PY3: + DATASET_ID = 'pydata_pandas_bq_testing_py3' +else: + DATASET_ID = 'pydata_pandas_bq_testing_py2' + +TABLE_ID = 'new_test' +DESTINATION_TABLE = "{0}.{1}".format(DATASET_ID + "1", TABLE_ID) + +VERSION = platform.python_version() + +_IMPORTS = False +_GOOGLE_API_CLIENT_INSTALLED = False +_GOOGLE_API_CLIENT_VALID_VERSION = False +_HTTPLIB2_INSTALLED = False +_SETUPTOOLS_INSTALLED = False + + +def _skip_if_no_project_id(): + if not _get_project_id(): + raise nose.SkipTest( + "Cannot run integration tests without a project id") + + +def _skip_if_no_private_key_path(): + if not _get_private_key_path(): + raise nose.SkipTest("Cannot run integration tests without a " + "private key json file path") + + +def _skip_if_no_private_key_contents(): + if not _get_private_key_contents(): + raise nose.SkipTest("Cannot run integration tests without a " + "private key json contents") + + +def _in_travis_environment(): + return 'TRAVIS_BUILD_DIR' in os.environ and \ + 'GBQ_PROJECT_ID' in os.environ + + +def _get_project_id(): + if _in_travis_environment(): + return os.environ.get('GBQ_PROJECT_ID') + else: + return PROJECT_ID + + +def _get_private_key_path(): + if _in_travis_environment(): + return os.path.join(*[os.environ.get('TRAVIS_BUILD_DIR'), 'ci', + 'travis_gbq.json']) + else: + return PRIVATE_KEY_JSON_PATH + + +def _get_private_key_contents(): + if _in_travis_environment(): + with open(os.path.join(*[os.environ.get('TRAVIS_BUILD_DIR'), 'ci', + 'travis_gbq.json'])) as f: + return f.read() + else: + return PRIVATE_KEY_JSON_CONTENTS + + +def _test_imports(): + global _GOOGLE_API_CLIENT_INSTALLED, _GOOGLE_API_CLIENT_VALID_VERSION, \ + _HTTPLIB2_INSTALLED, _SETUPTOOLS_INSTALLED + + try: + import pkg_resources + _SETUPTOOLS_INSTALLED = True + except ImportError: + _SETUPTOOLS_INSTALLED = False + + if compat.PY3: + google_api_minimum_version = '1.4.1' + else: + google_api_minimum_version = '1.2.0' + + if _SETUPTOOLS_INSTALLED: + try: + try: + from googleapiclient.discovery import build # noqa + from googleapiclient.errors import HttpError # noqa + except: + from apiclient.discovery import build # noqa + from apiclient.errors import HttpError # noqa + + from oauth2client.client import OAuth2WebServerFlow # noqa + from oauth2client.client import AccessTokenRefreshError # noqa + + from oauth2client.file import Storage # noqa + from oauth2client.tools import run_flow # noqa + _GOOGLE_API_CLIENT_INSTALLED = True + _GOOGLE_API_CLIENT_VERSION = pkg_resources.get_distribution( + 'google-api-python-client').version + + if (StrictVersion(_GOOGLE_API_CLIENT_VERSION) >= + StrictVersion(google_api_minimum_version)): + _GOOGLE_API_CLIENT_VALID_VERSION = True + + except ImportError: + _GOOGLE_API_CLIENT_INSTALLED = False + + try: + import httplib2 # noqa + _HTTPLIB2_INSTALLED = True + except ImportError: + _HTTPLIB2_INSTALLED = False + + if not _SETUPTOOLS_INSTALLED: + raise ImportError('Could not import pkg_resources (setuptools).') + + if not _GOOGLE_API_CLIENT_INSTALLED: + raise ImportError('Could not import Google API Client.') + + if not _GOOGLE_API_CLIENT_VALID_VERSION: + raise ImportError("pandas requires google-api-python-client >= {0} " + "for Google BigQuery support, " + "current version {1}" + .format(google_api_minimum_version, + _GOOGLE_API_CLIENT_VERSION)) + + if not _HTTPLIB2_INSTALLED: + raise ImportError( + "pandas requires httplib2 for Google BigQuery support") + + # Bug fix for https://github.com/pandas-dev/pandas/issues/12572 + # We need to know that a supported version of oauth2client is installed + # Test that either of the following is installed: + # - SignedJwtAssertionCredentials from oauth2client.client + # - ServiceAccountCredentials from oauth2client.service_account + # SignedJwtAssertionCredentials is available in oauthclient < 2.0.0 + # ServiceAccountCredentials is available in oauthclient >= 2.0.0 + oauth2client_v1 = True + oauth2client_v2 = True + + try: + from oauth2client.client import SignedJwtAssertionCredentials # noqa + except ImportError: + oauth2client_v1 = False + + try: + from oauth2client.service_account import ServiceAccountCredentials # noqa + except ImportError: + oauth2client_v2 = False + + if not oauth2client_v1 and not oauth2client_v2: + raise ImportError("Missing oauth2client required for BigQuery " + "service account support") + + +def _setup_common(): + try: + _test_imports() + except (ImportError, NotImplementedError) as import_exception: + raise nose.SkipTest(import_exception) + + if _in_travis_environment(): + logging.getLogger('oauth2client').setLevel(logging.ERROR) + logging.getLogger('apiclient').setLevel(logging.ERROR) + + +def _check_if_can_get_correct_default_credentials(): + # Checks if "Application Default Credentials" can be fetched + # from the environment the tests are running in. + # See Issue #13577 + + import httplib2 + try: + from googleapiclient.discovery import build + except ImportError: + from apiclient.discovery import build + try: + from oauth2client.client import GoogleCredentials + credentials = GoogleCredentials.get_application_default() + http = httplib2.Http() + http = credentials.authorize(http) + bigquery_service = build('bigquery', 'v2', http=http) + jobs = bigquery_service.jobs() + job_data = {'configuration': {'query': {'query': 'SELECT 1'}}} + jobs.insert(projectId=_get_project_id(), body=job_data).execute() + return True + except: + return False + + +def clean_gbq_environment(private_key=None): + dataset = gbq._Dataset(_get_project_id(), private_key=private_key) + + for i in range(1, 10): + if DATASET_ID + str(i) in dataset.datasets(): + dataset_id = DATASET_ID + str(i) + table = gbq._Table(_get_project_id(), dataset_id, + private_key=private_key) + for j in range(1, 20): + if TABLE_ID + str(j) in dataset.tables(dataset_id): + table.delete(TABLE_ID + str(j)) + + dataset.delete(dataset_id) + + +def make_mixed_dataframe_v2(test_size): + # create df to test for all BQ datatypes except RECORD + bools = np.random.randint(2, size=(1, test_size)).astype(bool) + flts = np.random.randn(1, test_size) + ints = np.random.randint(1, 10, size=(1, test_size)) + strs = np.random.randint(1, 10, size=(1, test_size)).astype(str) + times = [datetime.now(pytz.timezone('US/Arizona')) + for t in range(test_size)] + return DataFrame({'bools': bools[0], + 'flts': flts[0], + 'ints': ints[0], + 'strs': strs[0], + 'times': times[0]}, + index=range(test_size)) + + +def test_generate_bq_schema_deprecated(): + # 11121 Deprecation of generate_bq_schema + with tm.assert_produces_warning(FutureWarning): + df = make_mixed_dataframe_v2(10) + gbq.generate_bq_schema(df) + + +class TestGBQConnectorIntegration(tm.TestCase): + + def setUp(self): + _setup_common() + _skip_if_no_project_id() + + self.sut = gbq.GbqConnector(_get_project_id(), + private_key=_get_private_key_path()) + + def test_should_be_able_to_make_a_connector(self): + self.assertTrue(self.sut is not None, + 'Could not create a GbqConnector') + + def test_should_be_able_to_get_valid_credentials(self): + credentials = self.sut.get_credentials() + self.assertFalse(credentials.invalid, 'Returned credentials invalid') + + def test_should_be_able_to_get_a_bigquery_service(self): + bigquery_service = self.sut.get_service() + self.assertTrue(bigquery_service is not None, 'No service returned') + + def test_should_be_able_to_get_schema_from_query(self): + schema, pages = self.sut.run_query('SELECT 1') + self.assertTrue(schema is not None) + + def test_should_be_able_to_get_results_from_query(self): + schema, pages = self.sut.run_query('SELECT 1') + self.assertTrue(pages is not None) + + def test_get_application_default_credentials_does_not_throw_error(self): + if _check_if_can_get_correct_default_credentials(): + raise nose.SkipTest("Can get default_credentials " + "from the environment!") + credentials = self.sut.get_application_default_credentials() + self.assertIsNone(credentials) + + def test_get_application_default_credentials_returns_credentials(self): + if not _check_if_can_get_correct_default_credentials(): + raise nose.SkipTest("Cannot get default_credentials " + "from the environment!") + from oauth2client.client import GoogleCredentials + credentials = self.sut.get_application_default_credentials() + self.assertTrue(isinstance(credentials, GoogleCredentials)) + + +class TestGBQConnectorServiceAccountKeyPathIntegration(tm.TestCase): + + def setUp(self): + _setup_common() + + _skip_if_no_project_id() + _skip_if_no_private_key_path() + + self.sut = gbq.GbqConnector(_get_project_id(), + private_key=_get_private_key_path()) + + def test_should_be_able_to_make_a_connector(self): + self.assertTrue(self.sut is not None, + 'Could not create a GbqConnector') + + def test_should_be_able_to_get_valid_credentials(self): + credentials = self.sut.get_credentials() + self.assertFalse(credentials.invalid, 'Returned credentials invalid') + + def test_should_be_able_to_get_a_bigquery_service(self): + bigquery_service = self.sut.get_service() + self.assertTrue(bigquery_service is not None, 'No service returned') + + def test_should_be_able_to_get_schema_from_query(self): + schema, pages = self.sut.run_query('SELECT 1') + self.assertTrue(schema is not None) + + def test_should_be_able_to_get_results_from_query(self): + schema, pages = self.sut.run_query('SELECT 1') + self.assertTrue(pages is not None) + + +class TestGBQConnectorServiceAccountKeyContentsIntegration(tm.TestCase): + + def setUp(self): + _setup_common() + + _skip_if_no_project_id() + _skip_if_no_private_key_path() + + self.sut = gbq.GbqConnector(_get_project_id(), + private_key=_get_private_key_path()) + + def test_should_be_able_to_make_a_connector(self): + self.assertTrue(self.sut is not None, + 'Could not create a GbqConnector') + + def test_should_be_able_to_get_valid_credentials(self): + credentials = self.sut.get_credentials() + self.assertFalse(credentials.invalid, 'Returned credentials invalid') + + def test_should_be_able_to_get_a_bigquery_service(self): + bigquery_service = self.sut.get_service() + self.assertTrue(bigquery_service is not None, 'No service returned') + + def test_should_be_able_to_get_schema_from_query(self): + schema, pages = self.sut.run_query('SELECT 1') + self.assertTrue(schema is not None) + + def test_should_be_able_to_get_results_from_query(self): + schema, pages = self.sut.run_query('SELECT 1') + self.assertTrue(pages is not None) + + +class GBQUnitTests(tm.TestCase): + + def setUp(self): + _setup_common() + + def test_import_google_api_python_client(self): + if compat.PY2: + with tm.assertRaises(ImportError): + from googleapiclient.discovery import build # noqa + from googleapiclient.errors import HttpError # noqa + from apiclient.discovery import build # noqa + from apiclient.errors import HttpError # noqa + else: + from googleapiclient.discovery import build # noqa + from googleapiclient.errors import HttpError # noqa + + def test_should_return_bigquery_integers_as_python_floats(self): + result = gbq._parse_entry(1, 'INTEGER') + tm.assert_equal(result, float(1)) + + def test_should_return_bigquery_floats_as_python_floats(self): + result = gbq._parse_entry(1, 'FLOAT') + tm.assert_equal(result, float(1)) + + def test_should_return_bigquery_timestamps_as_numpy_datetime(self): + result = gbq._parse_entry('0e9', 'TIMESTAMP') + tm.assert_equal(result, np_datetime64_compat('1970-01-01T00:00:00Z')) + + def test_should_return_bigquery_booleans_as_python_booleans(self): + result = gbq._parse_entry('false', 'BOOLEAN') + tm.assert_equal(result, False) + + def test_should_return_bigquery_strings_as_python_strings(self): + result = gbq._parse_entry('STRING', 'STRING') + tm.assert_equal(result, 'STRING') + + def test_to_gbq_should_fail_if_invalid_table_name_passed(self): + with tm.assertRaises(gbq.NotFoundException): + gbq.to_gbq(DataFrame(), 'invalid_table_name', project_id="1234") + + def test_to_gbq_with_no_project_id_given_should_fail(self): + with tm.assertRaises(TypeError): + gbq.to_gbq(DataFrame(), 'dataset.tablename') + + def test_read_gbq_with_no_project_id_given_should_fail(self): + with tm.assertRaises(TypeError): + gbq.read_gbq('SELECT "1" as NUMBER_1') + + def test_that_parse_data_works_properly(self): + test_schema = {'fields': [ + {'mode': 'NULLABLE', 'name': 'VALID_STRING', 'type': 'STRING'}]} + test_page = [{'f': [{'v': 'PI'}]}] + + test_output = gbq._parse_data(test_schema, test_page) + correct_output = DataFrame({'VALID_STRING': ['PI']}) + tm.assert_frame_equal(test_output, correct_output) + + def test_read_gbq_with_invalid_private_key_json_should_fail(self): + with tm.assertRaises(gbq.InvalidPrivateKeyFormat): + gbq.read_gbq('SELECT 1', project_id='x', private_key='y') + + def test_read_gbq_with_empty_private_key_json_should_fail(self): + with tm.assertRaises(gbq.InvalidPrivateKeyFormat): + gbq.read_gbq('SELECT 1', project_id='x', private_key='{}') + + def test_read_gbq_with_private_key_json_wrong_types_should_fail(self): + with tm.assertRaises(gbq.InvalidPrivateKeyFormat): + gbq.read_gbq( + 'SELECT 1', project_id='x', + private_key='{ "client_email" : 1, "private_key" : True }') + + def test_read_gbq_with_empty_private_key_file_should_fail(self): + with tm.ensure_clean() as empty_file_path: + with tm.assertRaises(gbq.InvalidPrivateKeyFormat): + gbq.read_gbq('SELECT 1', project_id='x', + private_key=empty_file_path) + + def test_read_gbq_with_corrupted_private_key_json_should_fail(self): + _skip_if_no_private_key_path() + + with tm.assertRaises(gbq.InvalidPrivateKeyFormat): + gbq.read_gbq( + 'SELECT 1', project_id='x', + private_key=re.sub('[a-z]', '9', _get_private_key_path())) + + +class TestReadGBQIntegration(tm.TestCase): + + @classmethod + def setUpClass(cls): + # - GLOBAL CLASS FIXTURES - + # put here any instruction you want to execute only *ONCE* *BEFORE* + # executing *ALL* tests described below. + + _skip_if_no_project_id() + + _setup_common() + + def setUp(self): + # - PER-TEST FIXTURES - + # put here any instruction you want to be run *BEFORE* *EVERY* test is + # executed. + pass + + @classmethod + def tearDownClass(cls): + # - GLOBAL CLASS FIXTURES - + # put here any instruction you want to execute only *ONCE* *AFTER* + # executing all tests. + pass + + def tearDown(self): + # - PER-TEST FIXTURES - + # put here any instructions you want to be run *AFTER* *EVERY* test is + # executed. + pass + + def test_should_read_as_user_account(self): + if _in_travis_environment(): + raise nose.SkipTest("Cannot run local auth in travis environment") + + query = 'SELECT "PI" as VALID_STRING' + df = gbq.read_gbq(query, project_id=_get_project_id()) + tm.assert_frame_equal(df, DataFrame({'VALID_STRING': ['PI']})) + + def test_should_read_as_service_account_with_key_path(self): + _skip_if_no_private_key_path() + query = 'SELECT "PI" as VALID_STRING' + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) + tm.assert_frame_equal(df, DataFrame({'VALID_STRING': ['PI']})) + + def test_should_read_as_service_account_with_key_contents(self): + _skip_if_no_private_key_contents() + query = 'SELECT "PI" as VALID_STRING' + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_contents()) + tm.assert_frame_equal(df, DataFrame({'VALID_STRING': ['PI']})) + + def test_should_properly_handle_valid_strings(self): + query = 'SELECT "PI" as VALID_STRING' + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) + tm.assert_frame_equal(df, DataFrame({'VALID_STRING': ['PI']})) + + def test_should_properly_handle_empty_strings(self): + query = 'SELECT "" as EMPTY_STRING' + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) + tm.assert_frame_equal(df, DataFrame({'EMPTY_STRING': [""]})) + + def test_should_properly_handle_null_strings(self): + query = 'SELECT STRING(NULL) as NULL_STRING' + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) + tm.assert_frame_equal(df, DataFrame({'NULL_STRING': [None]})) + + def test_should_properly_handle_valid_integers(self): + query = 'SELECT INTEGER(3) as VALID_INTEGER' + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) + tm.assert_frame_equal(df, DataFrame({'VALID_INTEGER': [3]})) + + def test_should_properly_handle_null_integers(self): + query = 'SELECT INTEGER(NULL) as NULL_INTEGER' + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) + tm.assert_frame_equal(df, DataFrame({'NULL_INTEGER': [np.nan]})) + + def test_should_properly_handle_valid_floats(self): + query = 'SELECT PI() as VALID_FLOAT' + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) + tm.assert_frame_equal(df, DataFrame( + {'VALID_FLOAT': [3.141592653589793]})) + + def test_should_properly_handle_null_floats(self): + query = 'SELECT FLOAT(NULL) as NULL_FLOAT' + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) + tm.assert_frame_equal(df, DataFrame({'NULL_FLOAT': [np.nan]})) + + def test_should_properly_handle_timestamp_unix_epoch(self): + query = 'SELECT TIMESTAMP("1970-01-01 00:00:00") as UNIX_EPOCH' + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) + tm.assert_frame_equal(df, DataFrame( + {'UNIX_EPOCH': [np.datetime64('1970-01-01T00:00:00.000000Z')]})) + + def test_should_properly_handle_arbitrary_timestamp(self): + query = 'SELECT TIMESTAMP("2004-09-15 05:00:00") as VALID_TIMESTAMP' + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) + tm.assert_frame_equal(df, DataFrame({ + 'VALID_TIMESTAMP': [np.datetime64('2004-09-15T05:00:00.000000Z')] + })) + + def test_should_properly_handle_null_timestamp(self): + query = 'SELECT TIMESTAMP(NULL) as NULL_TIMESTAMP' + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) + tm.assert_frame_equal(df, DataFrame({'NULL_TIMESTAMP': [NaT]})) + + def test_should_properly_handle_true_boolean(self): + query = 'SELECT BOOLEAN(TRUE) as TRUE_BOOLEAN' + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) + tm.assert_frame_equal(df, DataFrame({'TRUE_BOOLEAN': [True]})) + + def test_should_properly_handle_false_boolean(self): + query = 'SELECT BOOLEAN(FALSE) as FALSE_BOOLEAN' + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) + tm.assert_frame_equal(df, DataFrame({'FALSE_BOOLEAN': [False]})) + + def test_should_properly_handle_null_boolean(self): + query = 'SELECT BOOLEAN(NULL) as NULL_BOOLEAN' + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) + tm.assert_frame_equal(df, DataFrame({'NULL_BOOLEAN': [None]})) + + def test_unicode_string_conversion_and_normalization(self): + correct_test_datatype = DataFrame( + {'UNICODE_STRING': [u("\xe9\xfc")]} + ) + + unicode_string = "\xc3\xa9\xc3\xbc" + + if compat.PY3: + unicode_string = unicode_string.encode('latin-1').decode('utf8') + + query = 'SELECT "{0}" as UNICODE_STRING'.format(unicode_string) + + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) + tm.assert_frame_equal(df, correct_test_datatype) + + def test_index_column(self): + query = "SELECT 'a' as STRING_1, 'b' as STRING_2" + result_frame = gbq.read_gbq(query, project_id=_get_project_id(), + index_col="STRING_1", + private_key=_get_private_key_path()) + correct_frame = DataFrame( + {'STRING_1': ['a'], 'STRING_2': ['b']}).set_index("STRING_1") + tm.assert_equal(result_frame.index.name, correct_frame.index.name) + + def test_column_order(self): + query = "SELECT 'a' as STRING_1, 'b' as STRING_2, 'c' as STRING_3" + col_order = ['STRING_3', 'STRING_1', 'STRING_2'] + result_frame = gbq.read_gbq(query, project_id=_get_project_id(), + col_order=col_order, + private_key=_get_private_key_path()) + correct_frame = DataFrame({'STRING_1': ['a'], 'STRING_2': [ + 'b'], 'STRING_3': ['c']})[col_order] + tm.assert_frame_equal(result_frame, correct_frame) + + def test_column_order_plus_index(self): + query = "SELECT 'a' as STRING_1, 'b' as STRING_2, 'c' as STRING_3" + col_order = ['STRING_3', 'STRING_2'] + result_frame = gbq.read_gbq(query, project_id=_get_project_id(), + index_col='STRING_1', col_order=col_order, + private_key=_get_private_key_path()) + correct_frame = DataFrame( + {'STRING_1': ['a'], 'STRING_2': ['b'], 'STRING_3': ['c']}) + correct_frame.set_index('STRING_1', inplace=True) + correct_frame = correct_frame[col_order] + tm.assert_frame_equal(result_frame, correct_frame) + + def test_malformed_query(self): + with tm.assertRaises(gbq.GenericGBQException): + gbq.read_gbq("SELCET * FORM [publicdata:samples.shakespeare]", + project_id=_get_project_id(), + private_key=_get_private_key_path()) + + def test_bad_project_id(self): + with tm.assertRaises(gbq.GenericGBQException): + gbq.read_gbq("SELECT 1", project_id='001', + private_key=_get_private_key_path()) + + def test_bad_table_name(self): + with tm.assertRaises(gbq.GenericGBQException): + gbq.read_gbq("SELECT * FROM [publicdata:samples.nope]", + project_id=_get_project_id(), + private_key=_get_private_key_path()) + + def test_download_dataset_larger_than_200k_rows(self): + test_size = 200005 + # Test for known BigQuery bug in datasets larger than 100k rows + # http://stackoverflow.com/questions/19145587/bq-py-not-paging-results + df = gbq.read_gbq("SELECT id FROM [publicdata:samples.wikipedia] " + "GROUP EACH BY id ORDER BY id ASC LIMIT {0}" + .format(test_size), + project_id=_get_project_id(), + private_key=_get_private_key_path()) + self.assertEqual(len(df.drop_duplicates()), test_size) + + def test_zero_rows(self): + # Bug fix for https://github.com/pandas-dev/pandas/issues/10273 + df = gbq.read_gbq("SELECT title, id " + "FROM [publicdata:samples.wikipedia] " + "WHERE timestamp=-9999999", + project_id=_get_project_id(), + private_key=_get_private_key_path()) + page_array = np.zeros( + (0,), dtype=[('title', object), ('id', np.dtype(float))]) + expected_result = DataFrame(page_array, columns=['title', 'id']) + self.assert_frame_equal(df, expected_result) + + def test_legacy_sql(self): + legacy_sql = "SELECT id FROM [publicdata.samples.wikipedia] LIMIT 10" + + # Test that a legacy sql statement fails when + # setting dialect='standard' + with tm.assertRaises(gbq.GenericGBQException): + gbq.read_gbq(legacy_sql, project_id=_get_project_id(), + dialect='standard', + private_key=_get_private_key_path()) + + # Test that a legacy sql statement succeeds when + # setting dialect='legacy' + df = gbq.read_gbq(legacy_sql, project_id=_get_project_id(), + dialect='legacy', + private_key=_get_private_key_path()) + self.assertEqual(len(df.drop_duplicates()), 10) + + def test_standard_sql(self): + standard_sql = "SELECT DISTINCT id FROM " \ + "`publicdata.samples.wikipedia` LIMIT 10" + + # Test that a standard sql statement fails when using + # the legacy SQL dialect (default value) + with tm.assertRaises(gbq.GenericGBQException): + gbq.read_gbq(standard_sql, project_id=_get_project_id(), + private_key=_get_private_key_path()) + + # Test that a standard sql statement succeeds when + # setting dialect='standard' + df = gbq.read_gbq(standard_sql, project_id=_get_project_id(), + dialect='standard', + private_key=_get_private_key_path()) + self.assertEqual(len(df.drop_duplicates()), 10) + + def test_invalid_option_for_sql_dialect(self): + sql_statement = "SELECT DISTINCT id FROM " \ + "`publicdata.samples.wikipedia` LIMIT 10" + + # Test that an invalid option for `dialect` raises ValueError + with tm.assertRaises(ValueError): + gbq.read_gbq(sql_statement, project_id=_get_project_id(), + dialect='invalid', + private_key=_get_private_key_path()) + + # Test that a correct option for dialect succeeds + # to make sure ValueError was due to invalid dialect + gbq.read_gbq(sql_statement, project_id=_get_project_id(), + dialect='standard', private_key=_get_private_key_path()) + + def test_query_with_parameters(self): + sql_statement = "SELECT @param1 + @param2 as VALID_RESULT" + config = { + 'query': { + "useLegacySql": False, + "parameterMode": "named", + "queryParameters": [ + { + "name": "param1", + "parameterType": { + "type": "INTEGER" + }, + "parameterValue": { + "value": 1 + } + }, + { + "name": "param2", + "parameterType": { + "type": "INTEGER" + }, + "parameterValue": { + "value": 2 + } + } + ] + } + } + # Test that a query that relies on parameters fails + # when parameters are not supplied via configuration + with tm.assertRaises(ValueError): + gbq.read_gbq(sql_statement, project_id=_get_project_id(), + private_key=_get_private_key_path()) + + # Test that the query is successful because we have supplied + # the correct query parameters via the 'config' option + df = gbq.read_gbq(sql_statement, project_id=_get_project_id(), + private_key=_get_private_key_path(), + configuration=config) + tm.assert_frame_equal(df, DataFrame({'VALID_RESULT': [3]})) + + def test_query_inside_configuration(self): + query_no_use = 'SELECT "PI_WRONG" as VALID_STRING' + query = 'SELECT "PI" as VALID_STRING' + config = { + 'query': { + "query": query, + "useQueryCache": False, + } + } + # Test that it can't pass query both + # inside config and as parameter + with tm.assertRaises(ValueError): + gbq.read_gbq(query_no_use, project_id=_get_project_id(), + private_key=_get_private_key_path(), + configuration=config) + + df = gbq.read_gbq(None, project_id=_get_project_id(), + private_key=_get_private_key_path(), + configuration=config) + tm.assert_frame_equal(df, DataFrame({'VALID_STRING': ['PI']})) + + def test_configuration_without_query(self): + sql_statement = 'SELECT 1' + config = { + 'copy': { + "sourceTable": { + "projectId": _get_project_id(), + "datasetId": "publicdata:samples", + "tableId": "wikipedia" + }, + "destinationTable": { + "projectId": _get_project_id(), + "datasetId": "publicdata:samples", + "tableId": "wikipedia_copied" + }, + } + } + # Test that only 'query' configurations are supported + # nor 'copy','load','extract' + with tm.assertRaises(ValueError): + gbq.read_gbq(sql_statement, project_id=_get_project_id(), + private_key=_get_private_key_path(), + configuration=config) + + +class TestToGBQIntegration(tm.TestCase): + # Changes to BigQuery table schema may take up to 2 minutes as of May 2015 + # As a workaround to this issue, each test should use a unique table name. + # Make sure to modify the for loop range in the tearDownClass when a new + # test is added See `Issue 191 + # `__ + + @classmethod + def setUpClass(cls): + # - GLOBAL CLASS FIXTURES - + # put here any instruction you want to execute only *ONCE* *BEFORE* + # executing *ALL* tests described below. + + _skip_if_no_project_id() + + _setup_common() + clean_gbq_environment(_get_private_key_path()) + + gbq._Dataset(_get_project_id(), + private_key=_get_private_key_path() + ).create(DATASET_ID + "1") + + def setUp(self): + # - PER-TEST FIXTURES - + # put here any instruction you want to be run *BEFORE* *EVERY* test is + # executed. + + self.dataset = gbq._Dataset(_get_project_id(), + private_key=_get_private_key_path()) + self.table = gbq._Table(_get_project_id(), DATASET_ID + "1", + private_key=_get_private_key_path()) + self.sut = gbq.GbqConnector(_get_project_id(), + private_key=_get_private_key_path()) + + @classmethod + def tearDownClass(cls): + # - GLOBAL CLASS FIXTURES - + # put here any instruction you want to execute only *ONCE* *AFTER* + # executing all tests. + + clean_gbq_environment(_get_private_key_path()) + + def tearDown(self): + # - PER-TEST FIXTURES - + # put here any instructions you want to be run *AFTER* *EVERY* test is + # executed. + pass + + def test_upload_data(self): + destination_table = DESTINATION_TABLE + "1" + + test_size = 20001 + df = make_mixed_dataframe_v2(test_size) + + gbq.to_gbq(df, destination_table, _get_project_id(), chunksize=10000, + private_key=_get_private_key_path()) + + sleep(30) # <- Curses Google!!! + + result = gbq.read_gbq("SELECT COUNT(*) as NUM_ROWS FROM {0}" + .format(destination_table), + project_id=_get_project_id(), + private_key=_get_private_key_path()) + self.assertEqual(result['NUM_ROWS'][0], test_size) + + def test_upload_data_if_table_exists_fail(self): + destination_table = DESTINATION_TABLE + "2" + + test_size = 10 + df = make_mixed_dataframe_v2(test_size) + self.table.create(TABLE_ID + "2", gbq._generate_bq_schema(df)) + + # Test the default value of if_exists is 'fail' + with tm.assertRaises(gbq.TableCreationError): + gbq.to_gbq(df, destination_table, _get_project_id(), + private_key=_get_private_key_path()) + + # Test the if_exists parameter with value 'fail' + with tm.assertRaises(gbq.TableCreationError): + gbq.to_gbq(df, destination_table, _get_project_id(), + if_exists='fail', private_key=_get_private_key_path()) + + def test_upload_data_if_table_exists_append(self): + destination_table = DESTINATION_TABLE + "3" + + test_size = 10 + df = make_mixed_dataframe_v2(test_size) + df_different_schema = tm.makeMixedDataFrame() + + # Initialize table with sample data + gbq.to_gbq(df, destination_table, _get_project_id(), chunksize=10000, + private_key=_get_private_key_path()) + + # Test the if_exists parameter with value 'append' + gbq.to_gbq(df, destination_table, _get_project_id(), + if_exists='append', private_key=_get_private_key_path()) + + sleep(30) # <- Curses Google!!! + + result = gbq.read_gbq("SELECT COUNT(*) as NUM_ROWS FROM {0}" + .format(destination_table), + project_id=_get_project_id(), + private_key=_get_private_key_path()) + self.assertEqual(result['NUM_ROWS'][0], test_size * 2) + + # Try inserting with a different schema, confirm failure + with tm.assertRaises(gbq.InvalidSchema): + gbq.to_gbq(df_different_schema, destination_table, + _get_project_id(), if_exists='append', + private_key=_get_private_key_path()) + + def test_upload_data_if_table_exists_replace(self): + + raise nose.SkipTest("buggy test") + + destination_table = DESTINATION_TABLE + "4" + + test_size = 10 + df = make_mixed_dataframe_v2(test_size) + df_different_schema = tm.makeMixedDataFrame() + + # Initialize table with sample data + gbq.to_gbq(df, destination_table, _get_project_id(), chunksize=10000, + private_key=_get_private_key_path()) + + # Test the if_exists parameter with the value 'replace'. + gbq.to_gbq(df_different_schema, destination_table, + _get_project_id(), if_exists='replace', + private_key=_get_private_key_path()) + + sleep(30) # <- Curses Google!!! + + result = gbq.read_gbq("SELECT COUNT(*) as NUM_ROWS FROM {0}" + .format(destination_table), + project_id=_get_project_id(), + private_key=_get_private_key_path()) + self.assertEqual(result['NUM_ROWS'][0], 5) + + @tm.slow + def test_google_upload_errors_should_raise_exception(self): + destination_table = DESTINATION_TABLE + "5" + + test_timestamp = datetime.now(pytz.timezone('US/Arizona')) + bad_df = DataFrame({'bools': [False, False], 'flts': [0.0, 1.0], + 'ints': [0, '1'], 'strs': ['a', 1], + 'times': [test_timestamp, test_timestamp]}, + index=range(2)) + + with tm.assertRaises(gbq.StreamingInsertError): + gbq.to_gbq(bad_df, destination_table, _get_project_id(), + verbose=True, private_key=_get_private_key_path()) + + def test_generate_schema(self): + df = tm.makeMixedDataFrame() + schema = gbq._generate_bq_schema(df) + + test_schema = {'fields': [{'name': 'A', 'type': 'FLOAT'}, + {'name': 'B', 'type': 'FLOAT'}, + {'name': 'C', 'type': 'STRING'}, + {'name': 'D', 'type': 'TIMESTAMP'}]} + + self.assertEqual(schema, test_schema) + + def test_create_table(self): + destination_table = TABLE_ID + "6" + test_schema = {'fields': [{'name': 'A', 'type': 'FLOAT'}, + {'name': 'B', 'type': 'FLOAT'}, + {'name': 'C', 'type': 'STRING'}, + {'name': 'D', 'type': 'TIMESTAMP'}]} + self.table.create(destination_table, test_schema) + self.assertTrue(self.table.exists(destination_table), + 'Expected table to exist') + + def test_table_does_not_exist(self): + self.assertTrue(not self.table.exists(TABLE_ID + "7"), + 'Expected table not to exist') + + def test_delete_table(self): + destination_table = TABLE_ID + "8" + test_schema = {'fields': [{'name': 'A', 'type': 'FLOAT'}, + {'name': 'B', 'type': 'FLOAT'}, + {'name': 'C', 'type': 'STRING'}, + {'name': 'D', 'type': 'TIMESTAMP'}]} + self.table.create(destination_table, test_schema) + self.table.delete(destination_table) + self.assertTrue(not self.table.exists( + destination_table), 'Expected table not to exist') + + def test_list_table(self): + destination_table = TABLE_ID + "9" + test_schema = {'fields': [{'name': 'A', 'type': 'FLOAT'}, + {'name': 'B', 'type': 'FLOAT'}, + {'name': 'C', 'type': 'STRING'}, + {'name': 'D', 'type': 'TIMESTAMP'}]} + self.table.create(destination_table, test_schema) + self.assertTrue( + destination_table in self.dataset.tables(DATASET_ID + "1"), + 'Expected table list to contain table {0}' + .format(destination_table)) + + def test_verify_schema_allows_flexible_column_order(self): + destination_table = TABLE_ID + "10" + test_schema_1 = {'fields': [{'name': 'A', 'type': 'FLOAT'}, + {'name': 'B', 'type': 'FLOAT'}, + {'name': 'C', 'type': 'STRING'}, + {'name': 'D', 'type': 'TIMESTAMP'}]} + test_schema_2 = {'fields': [{'name': 'A', 'type': 'FLOAT'}, + {'name': 'C', 'type': 'STRING'}, + {'name': 'B', 'type': 'FLOAT'}, + {'name': 'D', 'type': 'TIMESTAMP'}]} + + self.table.create(destination_table, test_schema_1) + self.assertTrue(self.sut.verify_schema( + DATASET_ID + "1", destination_table, test_schema_2), + 'Expected schema to match') + + def test_verify_schema_fails_different_data_type(self): + destination_table = TABLE_ID + "11" + test_schema_1 = {'fields': [{'name': 'A', 'type': 'FLOAT'}, + {'name': 'B', 'type': 'FLOAT'}, + {'name': 'C', 'type': 'STRING'}, + {'name': 'D', 'type': 'TIMESTAMP'}]} + test_schema_2 = {'fields': [{'name': 'A', 'type': 'FLOAT'}, + {'name': 'B', 'type': 'STRING'}, + {'name': 'C', 'type': 'STRING'}, + {'name': 'D', 'type': 'TIMESTAMP'}]} + + self.table.create(destination_table, test_schema_1) + self.assertFalse(self.sut.verify_schema( + DATASET_ID + "1", destination_table, test_schema_2), + 'Expected different schema') + + def test_verify_schema_fails_different_structure(self): + destination_table = TABLE_ID + "12" + test_schema_1 = {'fields': [{'name': 'A', 'type': 'FLOAT'}, + {'name': 'B', 'type': 'FLOAT'}, + {'name': 'C', 'type': 'STRING'}, + {'name': 'D', 'type': 'TIMESTAMP'}]} + test_schema_2 = {'fields': [{'name': 'A', 'type': 'FLOAT'}, + {'name': 'B2', 'type': 'FLOAT'}, + {'name': 'C', 'type': 'STRING'}, + {'name': 'D', 'type': 'TIMESTAMP'}]} + + self.table.create(destination_table, test_schema_1) + self.assertFalse(self.sut.verify_schema( + DATASET_ID + "1", destination_table, test_schema_2), + 'Expected different schema') + + def test_upload_data_flexible_column_order(self): + destination_table = DESTINATION_TABLE + "13" + + test_size = 10 + df = make_mixed_dataframe_v2(test_size) + + # Initialize table with sample data + gbq.to_gbq(df, destination_table, _get_project_id(), chunksize=10000, + private_key=_get_private_key_path()) + + df_columns_reversed = df[df.columns[::-1]] + + gbq.to_gbq(df_columns_reversed, destination_table, _get_project_id(), + if_exists='append', private_key=_get_private_key_path()) + + def test_list_dataset(self): + dataset_id = DATASET_ID + "1" + self.assertTrue(dataset_id in self.dataset.datasets(), + 'Expected dataset list to contain dataset {0}' + .format(dataset_id)) + + def test_list_table_zero_results(self): + dataset_id = DATASET_ID + "2" + self.dataset.create(dataset_id) + table_list = gbq._Dataset(_get_project_id(), + private_key=_get_private_key_path() + ).tables(dataset_id) + self.assertEqual(len(table_list), 0, + 'Expected gbq.list_table() to return 0') + + def test_create_dataset(self): + dataset_id = DATASET_ID + "3" + self.dataset.create(dataset_id) + self.assertTrue(dataset_id in self.dataset.datasets(), + 'Expected dataset to exist') + + def test_delete_dataset(self): + dataset_id = DATASET_ID + "4" + self.dataset.create(dataset_id) + self.dataset.delete(dataset_id) + self.assertTrue(dataset_id not in self.dataset.datasets(), + 'Expected dataset not to exist') + + def test_dataset_exists(self): + dataset_id = DATASET_ID + "5" + self.dataset.create(dataset_id) + self.assertTrue(self.dataset.exists(dataset_id), + 'Expected dataset to exist') + + def create_table_data_dataset_does_not_exist(self): + dataset_id = DATASET_ID + "6" + table_id = TABLE_ID + "1" + table_with_new_dataset = gbq._Table(_get_project_id(), dataset_id) + df = make_mixed_dataframe_v2(10) + table_with_new_dataset.create(table_id, gbq._generate_bq_schema(df)) + self.assertTrue(self.dataset.exists(dataset_id), + 'Expected dataset to exist') + self.assertTrue(table_with_new_dataset.exists( + table_id), 'Expected dataset to exist') + + def test_dataset_does_not_exist(self): + self.assertTrue(not self.dataset.exists( + DATASET_ID + "_not_found"), 'Expected dataset not to exist') + + +class TestToGBQIntegrationServiceAccountKeyPath(tm.TestCase): + # Changes to BigQuery table schema may take up to 2 minutes as of May 2015 + # As a workaround to this issue, each test should use a unique table name. + # Make sure to modify the for loop range in the tearDownClass when a new + # test is added + # See `Issue 191 + # `__ + + @classmethod + def setUpClass(cls): + # - GLOBAL CLASS FIXTURES - + # put here any instruction you want to execute only *ONCE* *BEFORE* + # executing *ALL* tests described below. + + _skip_if_no_project_id() + _skip_if_no_private_key_path() + + _setup_common() + clean_gbq_environment(_get_private_key_path()) + + def setUp(self): + # - PER-TEST FIXTURES - + # put here any instruction you want to be run *BEFORE* *EVERY* test + # is executed. + pass + + @classmethod + def tearDownClass(cls): + # - GLOBAL CLASS FIXTURES - + # put here any instruction you want to execute only *ONCE* *AFTER* + # executing all tests. + + clean_gbq_environment(_get_private_key_path()) + + def tearDown(self): + # - PER-TEST FIXTURES - + # put here any instructions you want to be run *AFTER* *EVERY* test + # is executed. + pass + + def test_upload_data_as_service_account_with_key_path(self): + destination_table = "{0}.{1}".format(DATASET_ID + "2", TABLE_ID + "1") + + test_size = 10 + df = make_mixed_dataframe_v2(test_size) + + gbq.to_gbq(df, destination_table, _get_project_id(), chunksize=10000, + private_key=_get_private_key_path()) + + sleep(30) # <- Curses Google!!! + + result = gbq.read_gbq( + "SELECT COUNT(*) as NUM_ROWS FROM {0}".format(destination_table), + project_id=_get_project_id(), + private_key=_get_private_key_path()) + + self.assertEqual(result['NUM_ROWS'][0], test_size) + + +class TestToGBQIntegrationServiceAccountKeyContents(tm.TestCase): + # Changes to BigQuery table schema may take up to 2 minutes as of May 2015 + # As a workaround to this issue, each test should use a unique table name. + # Make sure to modify the for loop range in the tearDownClass when a new + # test is added + # See `Issue 191 + # `__ + + @classmethod + def setUpClass(cls): + # - GLOBAL CLASS FIXTURES - + # put here any instruction you want to execute only *ONCE* *BEFORE* + # executing *ALL* tests described below. + + _setup_common() + _skip_if_no_project_id() + _skip_if_no_private_key_contents() + + clean_gbq_environment(_get_private_key_contents()) + + def setUp(self): + # - PER-TEST FIXTURES - + # put here any instruction you want to be run *BEFORE* *EVERY* test + # is executed. + pass + + @classmethod + def tearDownClass(cls): + # - GLOBAL CLASS FIXTURES - + # put here any instruction you want to execute only *ONCE* *AFTER* + # executing all tests. + + clean_gbq_environment(_get_private_key_contents()) + + def tearDown(self): + # - PER-TEST FIXTURES - + # put here any instructions you want to be run *AFTER* *EVERY* test + # is executed. + pass + + def test_upload_data_as_service_account_with_key_contents(self): + destination_table = "{0}.{1}".format(DATASET_ID + "3", TABLE_ID + "1") + + test_size = 10 + df = make_mixed_dataframe_v2(test_size) + + gbq.to_gbq(df, destination_table, _get_project_id(), chunksize=10000, + private_key=_get_private_key_contents()) + + sleep(30) # <- Curses Google!!! + + result = gbq.read_gbq( + "SELECT COUNT(*) as NUM_ROWS FROM {0}".format(destination_table), + project_id=_get_project_id(), + private_key=_get_private_key_contents()) + self.assertEqual(result['NUM_ROWS'][0], test_size) From 18853924b28d13f620771b855983d6d95c4eb8d2 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 8 Feb 2017 08:41:45 -0500 Subject: [PATCH 005/519] remove conda.recipe --- packages/pandas-gbq/conda.recipe/bld.bat | 8 ------- packages/pandas-gbq/conda.recipe/build.sh | 9 -------- packages/pandas-gbq/conda.recipe/meta.yaml | 27 ---------------------- 3 files changed, 44 deletions(-) delete mode 100644 packages/pandas-gbq/conda.recipe/bld.bat delete mode 100644 packages/pandas-gbq/conda.recipe/build.sh delete mode 100644 packages/pandas-gbq/conda.recipe/meta.yaml diff --git a/packages/pandas-gbq/conda.recipe/bld.bat b/packages/pandas-gbq/conda.recipe/bld.bat deleted file mode 100644 index 87b1481d740c..000000000000 --- a/packages/pandas-gbq/conda.recipe/bld.bat +++ /dev/null @@ -1,8 +0,0 @@ -"%PYTHON%" setup.py install -if errorlevel 1 exit 1 - -:: Add more build steps here, if they are necessary. - -:: See -:: http://docs.continuum.io/conda/build.html -:: for a list of environment variables that are set during the build process. diff --git a/packages/pandas-gbq/conda.recipe/build.sh b/packages/pandas-gbq/conda.recipe/build.sh deleted file mode 100644 index 4d7fc032b8cb..000000000000 --- a/packages/pandas-gbq/conda.recipe/build.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -$PYTHON setup.py install - -# Add more build steps here, if they are necessary. - -# See -# http://docs.continuum.io/conda/build.html -# for a list of environment variables that are set during the build process. diff --git a/packages/pandas-gbq/conda.recipe/meta.yaml b/packages/pandas-gbq/conda.recipe/meta.yaml deleted file mode 100644 index 57ba6095378e..000000000000 --- a/packages/pandas-gbq/conda.recipe/meta.yaml +++ /dev/null @@ -1,27 +0,0 @@ -package: - name: pandas-gbq - version: "0.1.0" - -source: - git_url: ../ - -requirements: - build: - - python - - setuptools - - run: - - python - - pandas - - httplib2 - - google-api-python-client - - oauth2client - -test: - # Python imports - imports: - - pandas_gbq -about: - home: https://github.com/pydata/pandas-gbq - license: BSD License - summary: 'pandas-gbq is a package providing an interface to the Google Big Query interface from pandas' From 114e62a5bd4d2749c8d916f13fe6c5f96f3fab31 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 8 Feb 2017 08:47:57 -0500 Subject: [PATCH 006/519] change nose skips to pytest skips --- .../pandas-gbq/pandas_gbq/tests/test_gbq.py | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py index 457e2d218cb3..33a4a2ad4008 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -1,6 +1,7 @@ +import pytest + import re from datetime import datetime -import nose import pytz import platform from time import sleep @@ -42,20 +43,20 @@ def _skip_if_no_project_id(): if not _get_project_id(): - raise nose.SkipTest( + pytest.skip( "Cannot run integration tests without a project id") def _skip_if_no_private_key_path(): if not _get_private_key_path(): - raise nose.SkipTest("Cannot run integration tests without a " - "private key json file path") + pytest.skip("Cannot run integration tests without a " + "private key json file path") def _skip_if_no_private_key_contents(): if not _get_private_key_contents(): - raise nose.SkipTest("Cannot run integration tests without a " - "private key json contents") + raise pytest.skip("Cannot run integration tests without a " + "private key json contents") def _in_travis_environment(): @@ -179,7 +180,7 @@ def _setup_common(): try: _test_imports() except (ImportError, NotImplementedError) as import_exception: - raise nose.SkipTest(import_exception) + pytest.skip(import_exception) if _in_travis_environment(): logging.getLogger('oauth2client').setLevel(logging.ERROR) @@ -279,15 +280,15 @@ def test_should_be_able_to_get_results_from_query(self): def test_get_application_default_credentials_does_not_throw_error(self): if _check_if_can_get_correct_default_credentials(): - raise nose.SkipTest("Can get default_credentials " - "from the environment!") + pytest.skip("Can get default_credentials " + "from the environment!") credentials = self.sut.get_application_default_credentials() self.assertIsNone(credentials) def test_get_application_default_credentials_returns_credentials(self): if not _check_if_can_get_correct_default_credentials(): - raise nose.SkipTest("Cannot get default_credentials " - "from the environment!") + pytest.skip("Cannot get default_credentials " + "from the environment!") from oauth2client.client import GoogleCredentials credentials = self.sut.get_application_default_credentials() self.assertTrue(isinstance(credentials, GoogleCredentials)) @@ -476,7 +477,7 @@ def tearDown(self): def test_should_read_as_user_account(self): if _in_travis_environment(): - raise nose.SkipTest("Cannot run local auth in travis environment") + pytest.skip("Cannot run local auth in travis environment") query = 'SELECT "PI" as VALID_STRING' df = gbq.read_gbq(query, project_id=_get_project_id()) @@ -913,7 +914,7 @@ def test_upload_data_if_table_exists_append(self): def test_upload_data_if_table_exists_replace(self): - raise nose.SkipTest("buggy test") + raise pytest.skip("buggy test") destination_table = DESTINATION_TABLE + "4" From 64055704fb2c5851d3e4807b582477d8ba90ae62 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 8 Feb 2017 08:56:43 -0500 Subject: [PATCH 007/519] update setup.py for requirements make tests run verbose --- packages/pandas-gbq/.travis.yml | 2 +- packages/pandas-gbq/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/.travis.yml b/packages/pandas-gbq/.travis.yml index bfed5a8acfeb..ed32085ed3d5 100644 --- a/packages/pandas-gbq/.travis.yml +++ b/packages/pandas-gbq/.travis.yml @@ -26,7 +26,7 @@ install: - python setup.py install script: - - pytest pandas_gbq + - pytest pandas_gbq -v - flake8 --version after_success: diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index d9d19642d007..c7dd71f2fcea 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -18,7 +18,7 @@ def readme(): return f.read() INSTALL_REQUIRES = ( - ['pandas', 'requests>=2.3.0', 'requests-file', 'requests-ftp'] + ['pandas', 'httplib2', 'google-api-python-client', 'oauth2client'] ) setup( From 6ba97bf45983e71a4012f7460e4c8a0623049355 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 8 Feb 2017 08:58:08 -0500 Subject: [PATCH 008/519] use pip installed flake8 --- packages/pandas-gbq/.travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/.travis.yml b/packages/pandas-gbq/.travis.yml index ed32085ed3d5..4aa8b13eb555 100644 --- a/packages/pandas-gbq/.travis.yml +++ b/packages/pandas-gbq/.travis.yml @@ -19,8 +19,8 @@ install: # Useful for debugging any issues with conda - conda info -a - - conda create -q -n test-environment python=$PYTHON pandas=$PANDAS pytest flake8 coverage setuptools - - pip install httplib2 google-api-python-client oauth2client pytest-cov + - conda create -q -n test-environment python=$PYTHON pandas=$PANDAS pytest coverage setuptools + - pip install httplib2 google-api-python-client oauth2client pytest-cov flake8 - source activate test-environment - conda list - python setup.py install From 0640f03acfa2e3edd074ffd3804cc1c9b60c6a17 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 8 Feb 2017 09:04:47 -0500 Subject: [PATCH 009/519] remove slow decorator (not used) --- packages/pandas-gbq/pandas_gbq/tests/test_gbq.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py index 33a4a2ad4008..2aea21e2f89b 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -939,7 +939,6 @@ def test_upload_data_if_table_exists_replace(self): private_key=_get_private_key_path()) self.assertEqual(result['NUM_ROWS'][0], 5) - @tm.slow def test_google_upload_errors_should_raise_exception(self): destination_table = DESTINATION_TABLE + "5" From a34e4316c787eef5c049d0c543859d23c1b1ec9e Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 8 Feb 2017 09:09:01 -0500 Subject: [PATCH 010/519] add per-build requirements --- packages/pandas-gbq/.travis.yml | 6 +++--- packages/pandas-gbq/ci/requirements-2.7.pip | 4 ++++ packages/pandas-gbq/ci/requirements-3.4.pip | 3 +++ packages/pandas-gbq/ci/requirements-3.5.pip | 3 +++ packages/pandas-gbq/ci/requirements-3.6.pip | 3 +++ 5 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 packages/pandas-gbq/ci/requirements-2.7.pip create mode 100644 packages/pandas-gbq/ci/requirements-3.4.pip create mode 100644 packages/pandas-gbq/ci/requirements-3.5.pip create mode 100644 packages/pandas-gbq/ci/requirements-3.6.pip diff --git a/packages/pandas-gbq/.travis.yml b/packages/pandas-gbq/.travis.yml index 4aa8b13eb555..e9696514d795 100644 --- a/packages/pandas-gbq/.travis.yml +++ b/packages/pandas-gbq/.travis.yml @@ -16,11 +16,11 @@ install: - conda config --set always_yes yes --set changeps1 no - conda config --add channels pandas - conda update -q conda - - # Useful for debugging any issues with conda - conda info -a - conda create -q -n test-environment python=$PYTHON pandas=$PANDAS pytest coverage setuptools - - pip install httplib2 google-api-python-client oauth2client pytest-cov flake8 + - pip install pytest-cov flake8 + - REQ="ci/requirements-${PYTHON_VERSION}.pip" + - pip install -r $REQ - source activate test-environment - conda list - python setup.py install diff --git a/packages/pandas-gbq/ci/requirements-2.7.pip b/packages/pandas-gbq/ci/requirements-2.7.pip new file mode 100644 index 000000000000..d48b405225aa --- /dev/null +++ b/packages/pandas-gbq/ci/requirements-2.7.pip @@ -0,0 +1,4 @@ +httplib2 +google-api-python-client==1.2 +python-gflags==2.0 +oauth2client==1.5.0 diff --git a/packages/pandas-gbq/ci/requirements-3.4.pip b/packages/pandas-gbq/ci/requirements-3.4.pip new file mode 100644 index 000000000000..05c938abcbab --- /dev/null +++ b/packages/pandas-gbq/ci/requirements-3.4.pip @@ -0,0 +1,3 @@ +httplib2 +google-api-python-client +oauth2client diff --git a/packages/pandas-gbq/ci/requirements-3.5.pip b/packages/pandas-gbq/ci/requirements-3.5.pip new file mode 100644 index 000000000000..05c938abcbab --- /dev/null +++ b/packages/pandas-gbq/ci/requirements-3.5.pip @@ -0,0 +1,3 @@ +httplib2 +google-api-python-client +oauth2client diff --git a/packages/pandas-gbq/ci/requirements-3.6.pip b/packages/pandas-gbq/ci/requirements-3.6.pip new file mode 100644 index 000000000000..05c938abcbab --- /dev/null +++ b/packages/pandas-gbq/ci/requirements-3.6.pip @@ -0,0 +1,3 @@ +httplib2 +google-api-python-client +oauth2client From a13b629105c6f00bad2eb4469b55bce8bb69f904 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 8 Feb 2017 09:10:33 -0500 Subject: [PATCH 011/519] requirements python version --- packages/pandas-gbq/.travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/.travis.yml b/packages/pandas-gbq/.travis.yml index e9696514d795..fda1c0c287c8 100644 --- a/packages/pandas-gbq/.travis.yml +++ b/packages/pandas-gbq/.travis.yml @@ -19,7 +19,7 @@ install: - conda info -a - conda create -q -n test-environment python=$PYTHON pandas=$PANDAS pytest coverage setuptools - pip install pytest-cov flake8 - - REQ="ci/requirements-${PYTHON_VERSION}.pip" + - REQ="ci/requirements-${PYTHON}.pip" - pip install -r $REQ - source activate test-environment - conda list From 790184a2b258c3dbacfac5cb3e96d506b80129e3 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 8 Feb 2017 09:27:02 -0500 Subject: [PATCH 012/519] make sure to install in activate environment --- packages/pandas-gbq/.travis.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/pandas-gbq/.travis.yml b/packages/pandas-gbq/.travis.yml index fda1c0c287c8..21189075fd5d 100644 --- a/packages/pandas-gbq/.travis.yml +++ b/packages/pandas-gbq/.travis.yml @@ -17,17 +17,18 @@ install: - conda config --add channels pandas - conda update -q conda - conda info -a - - conda create -q -n test-environment python=$PYTHON pandas=$PANDAS pytest coverage setuptools + - conda create -n test-environment python=$PYTHON + - source activate test-environment + - conda install pandas=$PANDAS pytest coverage - pip install pytest-cov flake8 - REQ="ci/requirements-${PYTHON}.pip" - pip install -r $REQ - - source activate test-environment - conda list - python setup.py install script: - - pytest pandas_gbq -v - - flake8 --version + - pytest pandas_gbq -v + - flake8 --version after_success: - coveralls From f29cc4ac9198ac955930677187882fa0d3abe843 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 8 Feb 2017 09:46:20 -0500 Subject: [PATCH 013/519] add coverage --- packages/pandas-gbq/.gitignore | 5 +++++ packages/pandas-gbq/.travis.yml | 18 +++++++++--------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/packages/pandas-gbq/.gitignore b/packages/pandas-gbq/.gitignore index 21b187c255ab..76d33fc77c1a 100644 --- a/packages/pandas-gbq/.gitignore +++ b/packages/pandas-gbq/.gitignore @@ -20,6 +20,11 @@ gi######################################### .ipynb_checkpoints .tags +# Coverage # +.coverage +coverage.xml +coverage_html_report + # Compiled source # ################### *.a diff --git a/packages/pandas-gbq/.travis.yml b/packages/pandas-gbq/.travis.yml index 21189075fd5d..80bb2913d6b4 100644 --- a/packages/pandas-gbq/.travis.yml +++ b/packages/pandas-gbq/.travis.yml @@ -3,10 +3,10 @@ sudo: false language: python env: - - PYTHON=2.7 PANDAS=0.19.2 - - PYTHON=3.4 PANDAS=0.18.1 - - PYTHON=3.5 PANDAS=0.19.2 - - PYTHON=3.6 PANDAS=0.19.2 + - PYTHON=2.7 PANDAS=0.19.2 COVERAGE='false' + - PYTHON=3.4 PANDAS=0.18.1 COVERAGE='false' + - PYTHON=3.5 PANDAS=0.19.2 COVERAGE='true' + - PYTHON=3.6 PANDAS=0.19.2 COVERAGE='false' install: - wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; @@ -19,16 +19,16 @@ install: - conda info -a - conda create -n test-environment python=$PYTHON - source activate test-environment - - conda install pandas=$PANDAS pytest coverage - - pip install pytest-cov flake8 + - conda install pandas=$PANDAS + - pip install coverage pytest pytest-cov flake8 - REQ="ci/requirements-${PYTHON}.pip" - pip install -r $REQ - conda list - python setup.py install script: - - pytest pandas_gbq -v - - flake8 --version + - pytest -s -v --cov=pandas_gbq --cov-report xml:/tmp/pytest.xml pandas_gbq + - flake8 pandas_gbq after_success: - - coveralls + - if [[ $COVERAGE == 'true' ]]; then coverage report --show-missing; pip install coveralls ; coveralls ; fi From 0f4c0b5ab3ef38d79994511166536a9f59eb601f Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 8 Feb 2017 09:56:32 -0500 Subject: [PATCH 014/519] flake errors --- packages/pandas-gbq/.travis.yml | 4 ++-- packages/pandas-gbq/pandas_gbq/gbq.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/.travis.yml b/packages/pandas-gbq/.travis.yml index 80bb2913d6b4..05e34c061273 100644 --- a/packages/pandas-gbq/.travis.yml +++ b/packages/pandas-gbq/.travis.yml @@ -27,8 +27,8 @@ install: - python setup.py install script: - - pytest -s -v --cov=pandas_gbq --cov-report xml:/tmp/pytest.xml pandas_gbq - - flake8 pandas_gbq + - pytest -s -v --cov=pandas_gbq --cov-report xml:/tmp/pytest-cov.xml pandas_gbq + - flake8 pandas_gbq -v after_success: - if [[ $COVERAGE == 'true' ]]; then coverage report --show-missing; pip install coveralls ; coveralls ; fi diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 966f53e9d75e..3741bf1adf19 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -60,6 +60,7 @@ def _test_google_api_imports(): raise ImportError("Missing module required for Google BigQuery " "support: {0}".format(str(e))) + logger = logging.getLogger('pandas.io.gbq') logger.setLevel(logging.ERROR) From c02bfd9bff0014c17f101d4764d9677665dde4ae Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 8 Feb 2017 10:08:41 -0500 Subject: [PATCH 015/519] use codecov --- packages/pandas-gbq/.travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/.travis.yml b/packages/pandas-gbq/.travis.yml index 05e34c061273..8c7acc0d3e6c 100644 --- a/packages/pandas-gbq/.travis.yml +++ b/packages/pandas-gbq/.travis.yml @@ -20,7 +20,7 @@ install: - conda create -n test-environment python=$PYTHON - source activate test-environment - conda install pandas=$PANDAS - - pip install coverage pytest pytest-cov flake8 + - pip install coverage pytest pytest-cov flake8 codecov - REQ="ci/requirements-${PYTHON}.pip" - pip install -r $REQ - conda list @@ -31,4 +31,4 @@ script: - flake8 pandas_gbq -v after_success: - - if [[ $COVERAGE == 'true' ]]; then coverage report --show-missing; pip install coveralls ; coveralls ; fi + - if [[ $COVERAGE == 'true' ]]; then codecov ; fi From ef02dea9ef4062f3f28174f423f6fbbac1c4e471 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 8 Feb 2017 10:35:07 -0500 Subject: [PATCH 016/519] doc build --- packages/pandas-gbq/.gitignore | 7 +- packages/pandas-gbq/docs/README.rst | 10 + .../pandas-gbq/docs/requirements-docs.txt | 5 + packages/pandas-gbq/docs/source/Makefile | 225 +++++++++++ packages/pandas-gbq/docs/source/conf.py | 349 ++++++++++++++++++ .../docs/source/examples-tutorials.rst | 2 + packages/pandas-gbq/docs/source/index.rst | 23 ++ packages/pandas-gbq/docs/source/install.rst | 27 ++ 8 files changed, 646 insertions(+), 2 deletions(-) create mode 100644 packages/pandas-gbq/docs/README.rst create mode 100644 packages/pandas-gbq/docs/requirements-docs.txt create mode 100644 packages/pandas-gbq/docs/source/Makefile create mode 100644 packages/pandas-gbq/docs/source/conf.py create mode 100644 packages/pandas-gbq/docs/source/examples-tutorials.rst create mode 100644 packages/pandas-gbq/docs/source/index.rst create mode 100644 packages/pandas-gbq/docs/source/install.rst diff --git a/packages/pandas-gbq/.gitignore b/packages/pandas-gbq/.gitignore index 76d33fc77c1a..eb19ab7b2a1e 100644 --- a/packages/pandas-gbq/.gitignore +++ b/packages/pandas-gbq/.gitignore @@ -20,7 +20,12 @@ gi######################################### .ipynb_checkpoints .tags +# Docs # +######## +docs/source/_build + # Coverage # +############ .coverage coverage.xml coverage_html_report @@ -43,8 +48,6 @@ MANIFEST ################ # setup.py working directory build -# sphinx build directory -doc/_build # setup.py dist directory dist # Egg metadata diff --git a/packages/pandas-gbq/docs/README.rst b/packages/pandas-gbq/docs/README.rst new file mode 100644 index 000000000000..8cd89d11da58 --- /dev/null +++ b/packages/pandas-gbq/docs/README.rst @@ -0,0 +1,10 @@ +To build a local copy of the pandas-gbq docs, install the programs in +requirements-docs.txt and run 'make html'. If you use the conda package manager +these commands suffice:: + + git clone git@github.com:pydata/pandas-gbq.git + cd dask/docs + conda create -n pandas-gbq-docs --file requirements-docs.txt + source activate pandas-gbq-docs + make html + open build/html/index.html diff --git a/packages/pandas-gbq/docs/requirements-docs.txt b/packages/pandas-gbq/docs/requirements-docs.txt new file mode 100644 index 000000000000..186b246693d9 --- /dev/null +++ b/packages/pandas-gbq/docs/requirements-docs.txt @@ -0,0 +1,5 @@ +numpydoc +sphinx +sphinx_rtd_theme +pandas +pandas-gbq diff --git a/packages/pandas-gbq/docs/source/Makefile b/packages/pandas-gbq/docs/source/Makefile new file mode 100644 index 000000000000..cd87dd5ef7a6 --- /dev/null +++ b/packages/pandas-gbq/docs/source/Makefile @@ -0,0 +1,225 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " applehelp to make an Apple Help Book" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " epub3 to make an epub3" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + @echo " coverage to run coverage check of the documentation (if enabled)" + @echo " dummy to check syntax errors of document sources" + +.PHONY: clean +clean: + rm -rf $(BUILDDIR)/* + +.PHONY: html +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +.PHONY: dirhtml +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +.PHONY: singlehtml +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +.PHONY: pickle +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +.PHONY: json +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +.PHONY: htmlhelp +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +.PHONY: qthelp +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/pandas-gbq.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/pandas-gbq.qhc" + +.PHONY: applehelp +applehelp: + $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp + @echo + @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." + @echo "N.B. You won't be able to view it unless you put it in" \ + "~/Library/Documentation/Help or install it in your application" \ + "bundle." + +.PHONY: devhelp +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/pandas-gbq" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/pandas-gbq" + @echo "# devhelp" + +.PHONY: epub +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +.PHONY: epub3 +epub3: + $(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3 + @echo + @echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3." + +.PHONY: latex +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +.PHONY: latexpdf +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: latexpdfja +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: text +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +.PHONY: man +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +.PHONY: texinfo +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +.PHONY: info +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +.PHONY: gettext +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +.PHONY: changes +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +.PHONY: linkcheck +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +.PHONY: doctest +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +.PHONY: coverage +coverage: + $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage + @echo "Testing of coverage in the sources finished, look at the " \ + "results in $(BUILDDIR)/coverage/python.txt." + +.PHONY: xml +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +.PHONY: pseudoxml +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." + +.PHONY: dummy +dummy: + $(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy + @echo + @echo "Build finished. Dummy builder generates no files." diff --git a/packages/pandas-gbq/docs/source/conf.py b/packages/pandas-gbq/docs/source/conf.py new file mode 100644 index 000000000000..81367a6201f1 --- /dev/null +++ b/packages/pandas-gbq/docs/source/conf.py @@ -0,0 +1,349 @@ +# -*- coding: utf-8 -*- +# +# pandas-gbq documentation build configuration file, created by +# sphinx-quickstart on Wed Feb 8 10:52:12 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.intersphinx', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The encoding of source files. +# +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'pandas-gbq' +copyright = u'2017, PyData Development Team' +author = u'PyData Development Team' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = u'0.1.0' +# The full version, including alpha/beta/rc tags. +release = u'0.1.0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# +# today = '' +# +# Else, today_fmt is used as the format for a strftime call. +# +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +# +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +# keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [] + +# The name for this set of Sphinx documents. +# " v documentation" by default. +# +# html_title = u'pandas-gbq v0.1.0' + +# A shorter title for the navigation bar. Default is the same as html_title. +# +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# +# html_logo = None + +# The name of an image file (relative to this directory) to use as a favicon of +# the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +# +# html_extra_path = [] + +# If not None, a 'Last updated on:' timestamp is inserted at every page +# bottom, using the given strftime format. +# The empty string is equivalent to '%b %d, %Y'. +# +# html_last_updated_fmt = None + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# +# html_additional_pages = {} + +# If false, no module index is generated. +# +# html_domain_indices = True + +# If false, no index is generated. +# +# html_use_index = True + +# If true, the index is split into individual pages for each letter. +# +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +# +# html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr', 'zh' +# +# html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# 'ja' uses this config value. +# 'zh' user can custom change `jieba` dictionary path. +# +# html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +# +# html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'pandas-gbqdoc' + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'pandas-gbq.tex', u'pandas-gbq Documentation', + u'PyData Development Team', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# +# latex_use_parts = False + +# If true, show page references after internal links. +# +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# +# latex_appendices = [] + +# It false, will not define \strong, \code, itleref, \crossref ... but only +# \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added +# packages. +# +# latex_keep_old_macro_names = True + +# If false, no module index is generated. +# +# latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'pandas-gbq', u'pandas-gbq Documentation', + [author], 1) +] + +# If true, show URL addresses after external links. +# +# man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'pandas-gbq', u'pandas-gbq Documentation', + author, 'pandas-gbq', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +# +# texinfo_appendices = [] + +# If false, no module index is generated. +# +# texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# +# texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +# +# texinfo_no_detailmenu = False + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {'https://docs.python.org/': None} + +extlinks = {'issue': ('https://github.com/pydata/pandas-gbq/issues/%s', + 'GH'), + 'wiki': ('https://github.com/pydata/pandas-gbq/wiki/%s', + 'wiki ')} diff --git a/packages/pandas-gbq/docs/source/examples-tutorials.rst b/packages/pandas-gbq/docs/source/examples-tutorials.rst new file mode 100644 index 000000000000..bac945d559fe --- /dev/null +++ b/packages/pandas-gbq/docs/source/examples-tutorials.rst @@ -0,0 +1,2 @@ +Examples +======== diff --git a/packages/pandas-gbq/docs/source/index.rst b/packages/pandas-gbq/docs/source/index.rst new file mode 100644 index 000000000000..cb6d4f56ae15 --- /dev/null +++ b/packages/pandas-gbq/docs/source/index.rst @@ -0,0 +1,23 @@ +.. pandas-gbq documentation master file, created by + sphinx-quickstart on Wed Feb 8 10:52:12 2017. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to pandas-gbq's documentation! +====================================== + +Contents: + +.. toctree:: + :maxdepth: 2 + + install.rst + examples-tutorials.rst + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/packages/pandas-gbq/docs/source/install.rst b/packages/pandas-gbq/docs/source/install.rst new file mode 100644 index 000000000000..b7dfd560831a --- /dev/null +++ b/packages/pandas-gbq/docs/source/install.rst @@ -0,0 +1,27 @@ +Install pandas-gbq +================== + +You can install pandas-gbq with ``conda``, with ``pip``, or by installing from source. + +Conda +----- + +To install the latest version of Dask from the +`conda-forge `_ repository using +`conda `_:: + + conda install pandas-gbq -c conda-forge + +This installs dask and all common dependencies, including Pandas and NumPy. + +Pip +--- + + +Install from Source +------------------- + + + +Test +---- From 8f98871532a1219a79466379e78c49eb002ef56e Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 14 Feb 2017 20:24:08 -0500 Subject: [PATCH 017/519] TST: Add pandas BigQuery credentials to enable integration testing (#9) --- packages/pandas-gbq/.travis.yml | 4 + packages/pandas-gbq/ci/requirements-2.7.pip | 1 + packages/pandas-gbq/ci/travis_encrypt_gbq.sh | 34 +++ packages/pandas-gbq/ci/travis_gbq.json.enc | Bin 0 -> 2352 bytes packages/pandas-gbq/ci/travis_gbq_config.txt | 2 + .../ci/travis_process_gbq_encryption.sh | 13 + packages/pandas-gbq/pandas_gbq/gbq.py | 83 ++++--- .../pandas-gbq/pandas_gbq/tests/test_gbq.py | 224 +++++++++--------- 8 files changed, 219 insertions(+), 142 deletions(-) create mode 100755 packages/pandas-gbq/ci/travis_encrypt_gbq.sh create mode 100644 packages/pandas-gbq/ci/travis_gbq.json.enc create mode 100644 packages/pandas-gbq/ci/travis_gbq_config.txt create mode 100644 packages/pandas-gbq/ci/travis_process_gbq_encryption.sh diff --git a/packages/pandas-gbq/.travis.yml b/packages/pandas-gbq/.travis.yml index 8c7acc0d3e6c..e2fd48f67133 100644 --- a/packages/pandas-gbq/.travis.yml +++ b/packages/pandas-gbq/.travis.yml @@ -8,6 +8,10 @@ env: - PYTHON=3.5 PANDAS=0.19.2 COVERAGE='true' - PYTHON=3.6 PANDAS=0.19.2 COVERAGE='false' +before_install: + - echo "before_install" + - source ci/travis_process_gbq_encryption.sh + install: - wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; - bash miniconda.sh -b -p $HOME/miniconda diff --git a/packages/pandas-gbq/ci/requirements-2.7.pip b/packages/pandas-gbq/ci/requirements-2.7.pip index d48b405225aa..103055ba7340 100644 --- a/packages/pandas-gbq/ci/requirements-2.7.pip +++ b/packages/pandas-gbq/ci/requirements-2.7.pip @@ -2,3 +2,4 @@ httplib2 google-api-python-client==1.2 python-gflags==2.0 oauth2client==1.5.0 +PyCrypto diff --git a/packages/pandas-gbq/ci/travis_encrypt_gbq.sh b/packages/pandas-gbq/ci/travis_encrypt_gbq.sh new file mode 100755 index 000000000000..e404ca73a405 --- /dev/null +++ b/packages/pandas-gbq/ci/travis_encrypt_gbq.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +GBQ_JSON_FILE=$1 + +if [[ $# -ne 1 ]]; then + echo -e "Too few arguments.\nUsage: ./travis_encrypt_gbq.sh "\ + "" + exit 1 +fi + +if [[ $GBQ_JSON_FILE != *.json ]]; then + echo "ERROR: Expected *.json file" + exit 1 +fi + +if [[ ! -f $GBQ_JSON_FILE ]]; then + echo "ERROR: File $GBQ_JSON_FILE does not exist" + exit 1 +fi + +echo "Encrypting $GBQ_JSON_FILE..." +read -d "\n" TRAVIS_KEY TRAVIS_IV <<<$(travis encrypt-file $GBQ_JSON_FILE \ +travis_gbq.json.enc -f | grep -o "\w*_iv\|\w*_key"); + +echo "Adding your secure key to travis_gbq_config.txt ..." +echo -e "TRAVIS_IV_ENV=$TRAVIS_IV\nTRAVIS_KEY_ENV=$TRAVIS_KEY"\ +> travis_gbq_config.txt + +echo "Done. Removing file $GBQ_JSON_FILE" +rm $GBQ_JSON_FILE + +echo -e "Created encrypted credentials file travis_gbq.json.enc.\n"\ + "NOTE: Do NOT commit the *.json file containing your unencrypted" \ + "private key" diff --git a/packages/pandas-gbq/ci/travis_gbq.json.enc b/packages/pandas-gbq/ci/travis_gbq.json.enc new file mode 100644 index 0000000000000000000000000000000000000000..6c6735b1a1489832c81393d3c6fb70db9335d2d4 GIT binary patch literal 2352 zcmV-03D5TNnqlXdY++)PWyH+_Fy*(K>9equQPR*sFH~Mv7?ngs_L)!gA#S%}I{l{Q zTNi$V9;3O4Z!v+`HNswsC?HD8;K91|kaWxdyIM^>oTz_m-uc5_#E)oKjY~K%0W+_ z{RHA}EvU@I6smY4s&Pv}R!FPGKJm6C)+Xod+@fD<>l7E_I%_#04fxtS<-e6nYZ!A} z_r18*Ck1zY4=na-VJhD!Oov>xZ4w zMPj0)WRya$gv)>0djJ$Y_mhE8yS)5V^`CYksDz)UGTg(^Zo9jeX|+f0RW8w_kDLgL zql4Mht+n8+((UU-@>iV}PLuE}ipFNb$?@5HDvMQQ0@|OAK-2ih|8Ys_@)u@GKc5X~ z3MxX!cC6)~gzV}Rgu@+^I@Kn6huy?c{7cDWS%C`1~T*OxX=SO6Dt*h}~|rwdTcW^0Apns~Sfy6iS$ zzN_)6lJ_KI5?}O4e|i@{bbiRq!mE@On6Y3j(8Mkx54!{5!g3HDE1Cqa#V;VfnRo8?1yB!TduBqgvSB6z@ClrR z;nWlk7X1z#igu=Ch+Z}KD{u7S-<@+N^b_D6t>j2u?>Q127|0797^ce6)}lqHC4cQn zb?WwjDojDqeEBO!M=Xd1K9!tb|3G~&^nN??789{rZH7pcW{b~&DlAE8J5Jm}lvci3 z$SGN%yhSWj2tEzAj&sIlMnTcUmZwsXByIsNBz;gsv%JjwL2{=Z;pUD8bbf~kG{~5S zA2|HLnRKgaSR?fj%>s#ftpcLsE)v#OFX?1Jee*y7YilpKP$K#e;Bes-PIu8~tD>7C zDvc^Gap1e)!A7K)H-k)`1Gk;{WX4f!Kx0l|ms`zT;6_WqBLqgS5j%&b35{rrC9q8R zP-yIY8@Co=8PSojukcIs)Mam^5t@;Pgh(As_!$gF3Ff5+DSq&#_9AHBS{FTSdFIcr zf{dVNI8)^N*8&ru&yT@cm$!_}k;demHQ3I_p$>1tAt+G!EX#ELXgw5ufF_-By!EFj z7-XM|pM!tCin1S-+3vXQGlLfwaQsGOmKGYxA$;L=JrC)xS36iw)5jf_A~eplLwG-g z(X+Do>(x@E1)Q@y(03L}(zkNR>LAUW+|<&{g7E`;TtcB&?s$$=z&EnV# zDn?FMIq>;sGI%E76J)~Nq~|ZyH<+_(4hfXZVWq|j`hrKL@iZUQmk8L%A9(j28ndMg zgL-$qWv>lzw4yT6ED()pFbuzB!f-GL+*ZH7dcUAEaBce1ECbuv4XOu`AQ#RxKj;sO zkrsdRtIpB)(uFI>U_Lq+9GbzW)?j&@9oesUn+?HqT^>@v^aU@j(fD`c8`{i;k?-|} zguupcc_rhf*}ZLM zaBP`mb>AK9l3wYKwHzX!6Ap0A+twAk(iKh5qH;! zm!Cbmr;eY&022xDpsFa#-!uKP(*k^)T`%v;f@2!bV9}qy-N=^+&w;7T`X!IOmbBW@ z^OC_L$GnbnZICluyyKVSfYbLnJ?gQ;o|^SO;TvKq_-)wNnl~jBNKpDorUby$qP+`v z5uMZ{RWGTSuxo5kYHPmAGV34^FF!%@SL^7Q_)ADuLH1U@>EYWjY9a&nq28F5=sza{ z3O?fO^%Q>o<$+EzkI5+6U>bI=RVYekMwJ{j<+-=jnp6+Gx9 zV)Gp@%eX)78ggwV@9%XL&KK;dacY9^-p~5sHM&GQnD=%R?iMt)CmWf>Y$Dy1=Zfvl WtYFY3cqn_L%Flv2;xfP?pVh*qxS|69 literal 0 HcmV?d00001 diff --git a/packages/pandas-gbq/ci/travis_gbq_config.txt b/packages/pandas-gbq/ci/travis_gbq_config.txt new file mode 100644 index 000000000000..fe22dbed8492 --- /dev/null +++ b/packages/pandas-gbq/ci/travis_gbq_config.txt @@ -0,0 +1,2 @@ +TRAVIS_IV_ENV=encrypted_53f0a3897064_iv +TRAVIS_KEY_ENV=encrypted_53f0a3897064_key diff --git a/packages/pandas-gbq/ci/travis_process_gbq_encryption.sh b/packages/pandas-gbq/ci/travis_process_gbq_encryption.sh new file mode 100644 index 000000000000..9967d40e49f0 --- /dev/null +++ b/packages/pandas-gbq/ci/travis_process_gbq_encryption.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +source ci/travis_gbq_config.txt + +if [[ -n ${SERVICE_ACCOUNT_KEY} ]]; then + echo "${SERVICE_ACCOUNT_KEY}" > ci/travis_gbq.json; +elif [[ -n ${!TRAVIS_IV_ENV} ]]; then + openssl aes-256-cbc -K ${!TRAVIS_KEY_ENV} -iv ${!TRAVIS_IV_ENV} \ + -in ci/travis_gbq.json.enc -out ci/travis_gbq.json -d; + export GBQ_PROJECT_ID='pandas-travis'; + echo 'Successfully decrypted gbq credentials' +fi + diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 3741bf1adf19..783e25bc3de9 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -955,8 +955,8 @@ def create(self, table_id, schema): """ if self.exists(table_id): - raise TableCreationError( - "The table could not be created because it already exists") + raise TableCreationError("Table {0} already " + "exists".format(table_id)) if not _Dataset(self.project_id, private_key=self.private_key).exists(self.dataset_id): @@ -1000,7 +1000,9 @@ def delete(self, table_id): projectId=self.project_id, tableId=table_id).execute() except self.http_error as ex: - self.process_http_error(ex) + # Ignore 404 error which may occur if table already deleted + if ex.resp.status != 404: + self.process_http_error(ex) class _Dataset(GbqConnector): @@ -1057,21 +1059,32 @@ def datasets(self): List of datasets under the specific project """ - try: - list_dataset_response = self.service.datasets().list( - projectId=self.project_id).execute().get('datasets', None) + dataset_list = [] + next_page_token = None + first_query = True - if not list_dataset_response: - return [] + while first_query or next_page_token: + first_query = False - dataset_list = list() + try: + list_dataset_response = self.service.datasets().list( + projectId=self.project_id, + pageToken=next_page_token).execute() - for row_num, raw_row in enumerate(list_dataset_response): - dataset_list.append(raw_row['datasetReference']['datasetId']) + dataset_response = list_dataset_response.get('datasets') + next_page_token = list_dataset_response.get('nextPageToken') - return dataset_list - except self.http_error as ex: - self.process_http_error(ex) + if not dataset_response: + return dataset_list + + for row_num, raw_row in enumerate(dataset_response): + dataset_list.append( + raw_row['datasetReference']['datasetId']) + + except self.http_error as ex: + self.process_http_error(ex) + + return dataset_list def create(self, dataset_id): """ Create a dataset in Google BigQuery @@ -1085,8 +1098,8 @@ def create(self, dataset_id): """ if self.exists(dataset_id): - raise DatasetCreationError( - "The dataset could not be created because it already exists") + raise DatasetCreationError("Dataset {0} already " + "exists".format(dataset_id)) body = { 'datasetReference': { @@ -1123,7 +1136,9 @@ def delete(self, dataset_id): projectId=self.project_id).execute() except self.http_error as ex: - self.process_http_error(ex) + # Ignore 404 error which may occur if dataset already deleted + if ex.resp.status != 404: + self.process_http_error(ex) def tables(self, dataset_id): """ List tables in the specific dataset in Google BigQuery @@ -1141,19 +1156,29 @@ def tables(self, dataset_id): List of tables under the specific dataset """ - try: - list_table_response = self.service.tables().list( - projectId=self.project_id, - datasetId=dataset_id).execute().get('tables', None) + table_list = [] + next_page_token = None + first_query = True - if not list_table_response: - return [] + while first_query or next_page_token: + first_query = False - table_list = list() + try: + list_table_response = self.service.tables().list( + projectId=self.project_id, + datasetId=dataset_id, + pageToken=next_page_token).execute() - for row_num, raw_row in enumerate(list_table_response): - table_list.append(raw_row['tableReference']['tableId']) + table_response = list_table_response.get('tables') + next_page_token = list_table_response.get('nextPageToken') - return table_list - except self.http_error as ex: - self.process_http_error(ex) + if not table_response: + return table_list + + for row_num, raw_row in enumerate(table_response): + table_list.append(raw_row['tableReference']['tableId']) + + except self.http_error as ex: + self.process_http_error(ex) + + return table_list diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py index 2aea21e2f89b..d9e6150f3052 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -3,9 +3,9 @@ import re from datetime import datetime import pytz -import platform from time import sleep import os +from random import randint import logging import numpy as np @@ -16,7 +16,7 @@ from pandas import NaT from pandas.compat import u, range from pandas.core.frame import DataFrame -import pandas.io.gbq as gbq +from pandas_gbq import gbq import pandas.util.testing as tm from pandas.compat.numpy import np_datetime64_compat @@ -24,15 +24,8 @@ PRIVATE_KEY_JSON_PATH = None PRIVATE_KEY_JSON_CONTENTS = None -if compat.PY3: - DATASET_ID = 'pydata_pandas_bq_testing_py3' -else: - DATASET_ID = 'pydata_pandas_bq_testing_py2' - TABLE_ID = 'new_test' -DESTINATION_TABLE = "{0}.{1}".format(DATASET_ID + "1", TABLE_ID) -VERSION = platform.python_version() _IMPORTS = False _GOOGLE_API_CLIENT_INSTALLED = False @@ -64,6 +57,10 @@ def _in_travis_environment(): 'GBQ_PROJECT_ID' in os.environ +def _get_dataset_prefix_random(): + return ''.join(['pandas_gbq_', str(randint(1, 100000))]) + + def _get_project_id(): if _in_travis_environment(): return os.environ.get('GBQ_PROJECT_ID') @@ -211,17 +208,17 @@ def _check_if_can_get_correct_default_credentials(): return False -def clean_gbq_environment(private_key=None): +def clean_gbq_environment(dataset_prefix, private_key=None): dataset = gbq._Dataset(_get_project_id(), private_key=private_key) - + all_datasets = dataset.datasets() for i in range(1, 10): - if DATASET_ID + str(i) in dataset.datasets(): - dataset_id = DATASET_ID + str(i) + dataset_id = dataset_prefix + str(i) + if dataset_id in all_datasets: table = gbq._Table(_get_project_id(), dataset_id, private_key=private_key) - for j in range(1, 20): - if TABLE_ID + str(j) in dataset.tables(dataset_id): - table.delete(TABLE_ID + str(j)) + all_tables = dataset.tables(dataset_id) + for table_id in all_tables: + table.delete(table_id) dataset.delete(dataset_id) @@ -364,6 +361,11 @@ def setUp(self): _setup_common() def test_import_google_api_python_client(self): + if not _in_travis_environment(): + pytest.skip("Skip if not in travis environment. Extra test to " + "make sure pandas_gbq doesn't break when " + "using google-api-python-client==1.2") + if compat.PY2: with tm.assertRaises(ImportError): from googleapiclient.discovery import build # noqa @@ -817,131 +819,124 @@ def setUpClass(cls): _skip_if_no_project_id() _setup_common() - clean_gbq_environment(_get_private_key_path()) - - gbq._Dataset(_get_project_id(), - private_key=_get_private_key_path() - ).create(DATASET_ID + "1") def setUp(self): # - PER-TEST FIXTURES - # put here any instruction you want to be run *BEFORE* *EVERY* test is # executed. + self.dataset_prefix = _get_dataset_prefix_random() + clean_gbq_environment(self.dataset_prefix, _get_private_key_path()) self.dataset = gbq._Dataset(_get_project_id(), private_key=_get_private_key_path()) - self.table = gbq._Table(_get_project_id(), DATASET_ID + "1", + self.table = gbq._Table(_get_project_id(), self.dataset_prefix + "1", private_key=_get_private_key_path()) self.sut = gbq.GbqConnector(_get_project_id(), private_key=_get_private_key_path()) + self.destination_table = "{0}{1}.{2}".format(self.dataset_prefix, "1", + TABLE_ID) + self.dataset.create(self.dataset_prefix + "1") @classmethod def tearDownClass(cls): # - GLOBAL CLASS FIXTURES - # put here any instruction you want to execute only *ONCE* *AFTER* # executing all tests. - - clean_gbq_environment(_get_private_key_path()) + pass def tearDown(self): # - PER-TEST FIXTURES - # put here any instructions you want to be run *AFTER* *EVERY* test is # executed. - pass + clean_gbq_environment(self.dataset_prefix, _get_private_key_path()) def test_upload_data(self): - destination_table = DESTINATION_TABLE + "1" - + test_id = "1" test_size = 20001 df = make_mixed_dataframe_v2(test_size) - gbq.to_gbq(df, destination_table, _get_project_id(), chunksize=10000, - private_key=_get_private_key_path()) + gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), + chunksize=10000, private_key=_get_private_key_path()) sleep(30) # <- Curses Google!!! result = gbq.read_gbq("SELECT COUNT(*) as NUM_ROWS FROM {0}" - .format(destination_table), + .format(self.destination_table + test_id), project_id=_get_project_id(), private_key=_get_private_key_path()) self.assertEqual(result['NUM_ROWS'][0], test_size) def test_upload_data_if_table_exists_fail(self): - destination_table = DESTINATION_TABLE + "2" - + test_id = "2" test_size = 10 df = make_mixed_dataframe_v2(test_size) - self.table.create(TABLE_ID + "2", gbq._generate_bq_schema(df)) + self.table.create(TABLE_ID + test_id, gbq._generate_bq_schema(df)) # Test the default value of if_exists is 'fail' with tm.assertRaises(gbq.TableCreationError): - gbq.to_gbq(df, destination_table, _get_project_id(), + gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), private_key=_get_private_key_path()) # Test the if_exists parameter with value 'fail' with tm.assertRaises(gbq.TableCreationError): - gbq.to_gbq(df, destination_table, _get_project_id(), + gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), if_exists='fail', private_key=_get_private_key_path()) def test_upload_data_if_table_exists_append(self): - destination_table = DESTINATION_TABLE + "3" - + test_id = "3" test_size = 10 df = make_mixed_dataframe_v2(test_size) df_different_schema = tm.makeMixedDataFrame() # Initialize table with sample data - gbq.to_gbq(df, destination_table, _get_project_id(), chunksize=10000, - private_key=_get_private_key_path()) + gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), + chunksize=10000, private_key=_get_private_key_path()) # Test the if_exists parameter with value 'append' - gbq.to_gbq(df, destination_table, _get_project_id(), + gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), if_exists='append', private_key=_get_private_key_path()) sleep(30) # <- Curses Google!!! result = gbq.read_gbq("SELECT COUNT(*) as NUM_ROWS FROM {0}" - .format(destination_table), + .format(self.destination_table + test_id), project_id=_get_project_id(), private_key=_get_private_key_path()) self.assertEqual(result['NUM_ROWS'][0], test_size * 2) # Try inserting with a different schema, confirm failure with tm.assertRaises(gbq.InvalidSchema): - gbq.to_gbq(df_different_schema, destination_table, + gbq.to_gbq(df_different_schema, self.destination_table + test_id, _get_project_id(), if_exists='append', private_key=_get_private_key_path()) def test_upload_data_if_table_exists_replace(self): - - raise pytest.skip("buggy test") - - destination_table = DESTINATION_TABLE + "4" - + test_id = "4" test_size = 10 df = make_mixed_dataframe_v2(test_size) df_different_schema = tm.makeMixedDataFrame() # Initialize table with sample data - gbq.to_gbq(df, destination_table, _get_project_id(), chunksize=10000, - private_key=_get_private_key_path()) + gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), + chunksize=10000, private_key=_get_private_key_path()) # Test the if_exists parameter with the value 'replace'. - gbq.to_gbq(df_different_schema, destination_table, + gbq.to_gbq(df_different_schema, self.destination_table + test_id, _get_project_id(), if_exists='replace', private_key=_get_private_key_path()) sleep(30) # <- Curses Google!!! result = gbq.read_gbq("SELECT COUNT(*) as NUM_ROWS FROM {0}" - .format(destination_table), + .format(self.destination_table + test_id), project_id=_get_project_id(), private_key=_get_private_key_path()) self.assertEqual(result['NUM_ROWS'][0], 5) def test_google_upload_errors_should_raise_exception(self): - destination_table = DESTINATION_TABLE + "5" + raise pytest.skip("buggy test") + test_id = "5" test_timestamp = datetime.now(pytz.timezone('US/Arizona')) bad_df = DataFrame({'bools': [False, False], 'flts': [0.0, 1.0], 'ints': [0, '1'], 'strs': ['a', 1], @@ -949,8 +944,8 @@ def test_google_upload_errors_should_raise_exception(self): index=range(2)) with tm.assertRaises(gbq.StreamingInsertError): - gbq.to_gbq(bad_df, destination_table, _get_project_id(), - verbose=True, private_key=_get_private_key_path()) + gbq.to_gbq(bad_df, self.destination_table + test_id, + _get_project_id(), private_key=_get_private_key_path()) def test_generate_schema(self): df = tm.makeMixedDataFrame() @@ -964,44 +959,45 @@ def test_generate_schema(self): self.assertEqual(schema, test_schema) def test_create_table(self): - destination_table = TABLE_ID + "6" + test_id = "6" test_schema = {'fields': [{'name': 'A', 'type': 'FLOAT'}, {'name': 'B', 'type': 'FLOAT'}, {'name': 'C', 'type': 'STRING'}, {'name': 'D', 'type': 'TIMESTAMP'}]} - self.table.create(destination_table, test_schema) - self.assertTrue(self.table.exists(destination_table), + self.table.create(TABLE_ID + test_id, test_schema) + self.assertTrue(self.table.exists(TABLE_ID + test_id), 'Expected table to exist') def test_table_does_not_exist(self): - self.assertTrue(not self.table.exists(TABLE_ID + "7"), + test_id = "7" + self.assertTrue(not self.table.exists(TABLE_ID + test_id), 'Expected table not to exist') def test_delete_table(self): - destination_table = TABLE_ID + "8" + test_id = "8" test_schema = {'fields': [{'name': 'A', 'type': 'FLOAT'}, {'name': 'B', 'type': 'FLOAT'}, {'name': 'C', 'type': 'STRING'}, {'name': 'D', 'type': 'TIMESTAMP'}]} - self.table.create(destination_table, test_schema) - self.table.delete(destination_table) + self.table.create(TABLE_ID + test_id, test_schema) + self.table.delete(TABLE_ID + test_id) self.assertTrue(not self.table.exists( - destination_table), 'Expected table not to exist') + TABLE_ID + test_id), 'Expected table not to exist') def test_list_table(self): - destination_table = TABLE_ID + "9" + test_id = "9" test_schema = {'fields': [{'name': 'A', 'type': 'FLOAT'}, {'name': 'B', 'type': 'FLOAT'}, {'name': 'C', 'type': 'STRING'}, {'name': 'D', 'type': 'TIMESTAMP'}]} - self.table.create(destination_table, test_schema) - self.assertTrue( - destination_table in self.dataset.tables(DATASET_ID + "1"), - 'Expected table list to contain table {0}' - .format(destination_table)) + self.table.create(TABLE_ID + test_id, test_schema) + self.assertTrue(TABLE_ID + test_id in + self.dataset.tables(self.dataset_prefix + "1"), + 'Expected table list to contain table {0}' + .format(TABLE_ID + test_id)) def test_verify_schema_allows_flexible_column_order(self): - destination_table = TABLE_ID + "10" + test_id = "10" test_schema_1 = {'fields': [{'name': 'A', 'type': 'FLOAT'}, {'name': 'B', 'type': 'FLOAT'}, {'name': 'C', 'type': 'STRING'}, @@ -1011,13 +1007,13 @@ def test_verify_schema_allows_flexible_column_order(self): {'name': 'B', 'type': 'FLOAT'}, {'name': 'D', 'type': 'TIMESTAMP'}]} - self.table.create(destination_table, test_schema_1) + self.table.create(TABLE_ID + test_id, test_schema_1) self.assertTrue(self.sut.verify_schema( - DATASET_ID + "1", destination_table, test_schema_2), + self.dataset_prefix + "1", TABLE_ID + test_id, test_schema_2), 'Expected schema to match') def test_verify_schema_fails_different_data_type(self): - destination_table = TABLE_ID + "11" + test_id = "11" test_schema_1 = {'fields': [{'name': 'A', 'type': 'FLOAT'}, {'name': 'B', 'type': 'FLOAT'}, {'name': 'C', 'type': 'STRING'}, @@ -1027,13 +1023,13 @@ def test_verify_schema_fails_different_data_type(self): {'name': 'C', 'type': 'STRING'}, {'name': 'D', 'type': 'TIMESTAMP'}]} - self.table.create(destination_table, test_schema_1) + self.table.create(TABLE_ID + test_id, test_schema_1) self.assertFalse(self.sut.verify_schema( - DATASET_ID + "1", destination_table, test_schema_2), + self.dataset_prefix + "1", TABLE_ID + test_id, test_schema_2), 'Expected different schema') def test_verify_schema_fails_different_structure(self): - destination_table = TABLE_ID + "12" + test_id = "12" test_schema_1 = {'fields': [{'name': 'A', 'type': 'FLOAT'}, {'name': 'B', 'type': 'FLOAT'}, {'name': 'C', 'type': 'STRING'}, @@ -1043,34 +1039,34 @@ def test_verify_schema_fails_different_structure(self): {'name': 'C', 'type': 'STRING'}, {'name': 'D', 'type': 'TIMESTAMP'}]} - self.table.create(destination_table, test_schema_1) + self.table.create(TABLE_ID + test_id, test_schema_1) self.assertFalse(self.sut.verify_schema( - DATASET_ID + "1", destination_table, test_schema_2), + self.dataset_prefix + "1", TABLE_ID + test_id, test_schema_2), 'Expected different schema') def test_upload_data_flexible_column_order(self): - destination_table = DESTINATION_TABLE + "13" - + test_id = "13" test_size = 10 df = make_mixed_dataframe_v2(test_size) # Initialize table with sample data - gbq.to_gbq(df, destination_table, _get_project_id(), chunksize=10000, - private_key=_get_private_key_path()) + gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), + chunksize=10000, private_key=_get_private_key_path()) df_columns_reversed = df[df.columns[::-1]] - gbq.to_gbq(df_columns_reversed, destination_table, _get_project_id(), - if_exists='append', private_key=_get_private_key_path()) + gbq.to_gbq(df_columns_reversed, self.destination_table + test_id, + _get_project_id(), if_exists='append', + private_key=_get_private_key_path()) def test_list_dataset(self): - dataset_id = DATASET_ID + "1" + dataset_id = self.dataset_prefix + "1" self.assertTrue(dataset_id in self.dataset.datasets(), 'Expected dataset list to contain dataset {0}' .format(dataset_id)) def test_list_table_zero_results(self): - dataset_id = DATASET_ID + "2" + dataset_id = self.dataset_prefix + "2" self.dataset.create(dataset_id) table_list = gbq._Dataset(_get_project_id(), private_key=_get_private_key_path() @@ -1079,26 +1075,26 @@ def test_list_table_zero_results(self): 'Expected gbq.list_table() to return 0') def test_create_dataset(self): - dataset_id = DATASET_ID + "3" + dataset_id = self.dataset_prefix + "3" self.dataset.create(dataset_id) self.assertTrue(dataset_id in self.dataset.datasets(), 'Expected dataset to exist') def test_delete_dataset(self): - dataset_id = DATASET_ID + "4" + dataset_id = self.dataset_prefix + "4" self.dataset.create(dataset_id) self.dataset.delete(dataset_id) self.assertTrue(dataset_id not in self.dataset.datasets(), 'Expected dataset not to exist') def test_dataset_exists(self): - dataset_id = DATASET_ID + "5" + dataset_id = self.dataset_prefix + "5" self.dataset.create(dataset_id) self.assertTrue(self.dataset.exists(dataset_id), 'Expected dataset to exist') def create_table_data_dataset_does_not_exist(self): - dataset_id = DATASET_ID + "6" + dataset_id = self.dataset_prefix + "6" table_id = TABLE_ID + "1" table_with_new_dataset = gbq._Table(_get_project_id(), dataset_id) df = make_mixed_dataframe_v2(10) @@ -1110,7 +1106,8 @@ def create_table_data_dataset_does_not_exist(self): def test_dataset_does_not_exist(self): self.assertTrue(not self.dataset.exists( - DATASET_ID + "_not_found"), 'Expected dataset not to exist') + self.dataset_prefix + "_not_found"), + 'Expected dataset not to exist') class TestToGBQIntegrationServiceAccountKeyPath(tm.TestCase): @@ -1131,41 +1128,42 @@ def setUpClass(cls): _skip_if_no_private_key_path() _setup_common() - clean_gbq_environment(_get_private_key_path()) def setUp(self): # - PER-TEST FIXTURES - # put here any instruction you want to be run *BEFORE* *EVERY* test # is executed. - pass + + self.dataset_prefix = _get_dataset_prefix_random() + clean_gbq_environment(self.dataset_prefix, _get_private_key_path()) + self.destination_table = "{0}{1}.{2}".format(self.dataset_prefix, "2", + TABLE_ID) @classmethod def tearDownClass(cls): # - GLOBAL CLASS FIXTURES - # put here any instruction you want to execute only *ONCE* *AFTER* # executing all tests. - - clean_gbq_environment(_get_private_key_path()) + pass def tearDown(self): # - PER-TEST FIXTURES - # put here any instructions you want to be run *AFTER* *EVERY* test # is executed. - pass + clean_gbq_environment(self.dataset_prefix, _get_private_key_path()) def test_upload_data_as_service_account_with_key_path(self): - destination_table = "{0}.{1}".format(DATASET_ID + "2", TABLE_ID + "1") - + test_id = "1" test_size = 10 df = make_mixed_dataframe_v2(test_size) - gbq.to_gbq(df, destination_table, _get_project_id(), chunksize=10000, - private_key=_get_private_key_path()) + gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), + chunksize=10000, private_key=_get_private_key_path()) sleep(30) # <- Curses Google!!! - result = gbq.read_gbq( - "SELECT COUNT(*) as NUM_ROWS FROM {0}".format(destination_table), + result = gbq.read_gbq("SELECT COUNT(*) as NUM_ROWS FROM {0}".format( + self.destination_table + test_id), project_id=_get_project_id(), private_key=_get_private_key_path()) @@ -1188,43 +1186,43 @@ def setUpClass(cls): _setup_common() _skip_if_no_project_id() - _skip_if_no_private_key_contents() - clean_gbq_environment(_get_private_key_contents()) + _skip_if_no_private_key_contents() def setUp(self): # - PER-TEST FIXTURES - # put here any instruction you want to be run *BEFORE* *EVERY* test # is executed. - pass + self.dataset_prefix = _get_dataset_prefix_random() + clean_gbq_environment(self.dataset_prefix, _get_private_key_contents()) + self.destination_table = "{0}{1}.{2}".format(self.dataset_prefix, "3", + TABLE_ID) @classmethod def tearDownClass(cls): # - GLOBAL CLASS FIXTURES - # put here any instruction you want to execute only *ONCE* *AFTER* # executing all tests. - - clean_gbq_environment(_get_private_key_contents()) + pass def tearDown(self): # - PER-TEST FIXTURES - # put here any instructions you want to be run *AFTER* *EVERY* test # is executed. - pass + clean_gbq_environment(self.dataset_prefix, _get_private_key_contents()) def test_upload_data_as_service_account_with_key_contents(self): - destination_table = "{0}.{1}".format(DATASET_ID + "3", TABLE_ID + "1") - + test_id = "1" test_size = 10 df = make_mixed_dataframe_v2(test_size) - gbq.to_gbq(df, destination_table, _get_project_id(), chunksize=10000, - private_key=_get_private_key_contents()) + gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), + chunksize=10000, private_key=_get_private_key_contents()) sleep(30) # <- Curses Google!!! - result = gbq.read_gbq( - "SELECT COUNT(*) as NUM_ROWS FROM {0}".format(destination_table), + result = gbq.read_gbq("SELECT COUNT(*) as NUM_ROWS FROM {0}".format( + self.destination_table + test_id), project_id=_get_project_id(), private_key=_get_private_key_contents()) self.assertEqual(result['NUM_ROWS'][0], test_size) From 0ba23a994b5973b3805133cc520dd2f353e8c9ea Mon Sep 17 00:00:00 2001 From: Piotr Chromiec Date: Wed, 15 Feb 2017 14:35:59 +0100 Subject: [PATCH 018/519] BUG: fix read_gbq lost numeric precision (#10) fixes: - lost precision for longs above 2^53 - and floats above 10k --- packages/pandas-gbq/pandas_gbq/gbq.py | 29 +- .../pandas-gbq/pandas_gbq/tests/test_gbq.py | 285 ++++++++++++------ 2 files changed, 206 insertions(+), 108 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 783e25bc3de9..64985dbd67a1 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -1,7 +1,6 @@ import warnings from datetime import datetime import json -import logging from time import sleep import uuid import time @@ -61,10 +60,6 @@ def _test_google_api_imports(): "support: {0}".format(str(e))) -logger = logging.getLogger('pandas.io.gbq') -logger.setLevel(logging.ERROR) - - class InvalidPrivateKeyFormat(PandasError, ValueError): """ Raised when provided private key has invalid format. @@ -604,18 +599,14 @@ def _parse_data(schema, rows): # see: # http://pandas.pydata.org/pandas-docs/dev/missing_data.html # #missing-data-casting-rules-and-indexing - dtype_map = {'INTEGER': np.dtype(float), - 'FLOAT': np.dtype(float), - # This seems to be buggy without nanosecond indicator + dtype_map = {'FLOAT': np.dtype(float), 'TIMESTAMP': 'M8[ns]'} fields = schema['fields'] col_types = [field['type'] for field in fields] col_names = [str(field['name']) for field in fields] col_dtypes = [dtype_map.get(field['type'], object) for field in fields] - page_array = np.zeros((len(rows),), - dtype=lzip(col_names, col_dtypes)) - + page_array = np.zeros((len(rows),), dtype=lzip(col_names, col_dtypes)) for row_num, raw_row in enumerate(rows): entries = raw_row.get('f', []) for col_num, field_type in enumerate(col_types): @@ -629,7 +620,9 @@ def _parse_data(schema, rows): def _parse_entry(field_value, field_type): if field_value is None or field_value == 'null': return None - if field_type == 'INTEGER' or field_type == 'FLOAT': + if field_type == 'INTEGER': + return int(field_value) + elif field_type == 'FLOAT': return float(field_value) elif field_type == 'TIMESTAMP': timestamp = datetime.utcfromtimestamp(float(field_value)) @@ -758,10 +751,14 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, 'Column order does not match this DataFrame.' ) - # Downcast floats to integers and objects to booleans - # if there are no NaN's. This is presently due to a - # limitation of numpy in handling missing data. - final_df._data = final_df._data.downcast(dtypes='infer') + # cast BOOLEAN and INTEGER columns from object to bool/int + # if they dont have any nulls + type_map = {'BOOLEAN': bool, 'INTEGER': int} + for field in schema['fields']: + if field['type'] in type_map and \ + final_df[field['name']].notnull().all(): + final_df[field['name']] = \ + final_df[field['name']].astype(type_map[field['type']]) connector.print_elapsed_seconds( 'Total time taken', diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py index d9e6150f3052..036e833051f1 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -40,6 +40,11 @@ def _skip_if_no_project_id(): "Cannot run integration tests without a project id") +def _skip_local_auth_if_in_travis_env(): + if _in_travis_environment(): + pytest.skip("Cannot run local auth in travis environment") + + def _skip_if_no_private_key_path(): if not _get_private_key_path(): pytest.skip("Cannot run integration tests without a " @@ -246,14 +251,14 @@ def test_generate_bq_schema_deprecated(): gbq.generate_bq_schema(df) -class TestGBQConnectorIntegration(tm.TestCase): +class TestGBQConnectorIntegrationWithLocalUserAccountAuth(tm.TestCase): def setUp(self): _setup_common() _skip_if_no_project_id() + _skip_local_auth_if_in_travis_env() - self.sut = gbq.GbqConnector(_get_project_id(), - private_key=_get_private_key_path()) + self.sut = gbq.GbqConnector(_get_project_id()) def test_should_be_able_to_make_a_connector(self): self.assertTrue(self.sut is not None, @@ -291,8 +296,7 @@ def test_get_application_default_credentials_returns_credentials(self): self.assertTrue(isinstance(credentials, GoogleCredentials)) -class TestGBQConnectorServiceAccountKeyPathIntegration(tm.TestCase): - +class TestGBQConnectorIntegrationWithServiceAccountKeyPath(tm.TestCase): def setUp(self): _setup_common() @@ -323,16 +327,15 @@ def test_should_be_able_to_get_results_from_query(self): self.assertTrue(pages is not None) -class TestGBQConnectorServiceAccountKeyContentsIntegration(tm.TestCase): - +class TestGBQConnectorIntegrationWithServiceAccountKeyContents(tm.TestCase): def setUp(self): _setup_common() _skip_if_no_project_id() - _skip_if_no_private_key_path() + _skip_if_no_private_key_contents() self.sut = gbq.GbqConnector(_get_project_id(), - private_key=_get_private_key_path()) + private_key=_get_private_key_contents()) def test_should_be_able_to_make_a_connector(self): self.assertTrue(self.sut is not None, @@ -376,9 +379,9 @@ def test_import_google_api_python_client(self): from googleapiclient.discovery import build # noqa from googleapiclient.errors import HttpError # noqa - def test_should_return_bigquery_integers_as_python_floats(self): + def test_should_return_bigquery_integers_as_python_ints(self): result = gbq._parse_entry(1, 'INTEGER') - tm.assert_equal(result, float(1)) + tm.assert_equal(result, int(1)) def test_should_return_bigquery_floats_as_python_floats(self): result = gbq._parse_entry(1, 'FLOAT') @@ -406,15 +409,15 @@ def test_to_gbq_with_no_project_id_given_should_fail(self): def test_read_gbq_with_no_project_id_given_should_fail(self): with tm.assertRaises(TypeError): - gbq.read_gbq('SELECT "1" as NUMBER_1') + gbq.read_gbq('SELECT 1') def test_that_parse_data_works_properly(self): test_schema = {'fields': [ - {'mode': 'NULLABLE', 'name': 'VALID_STRING', 'type': 'STRING'}]} + {'mode': 'NULLABLE', 'name': 'valid_string', 'type': 'STRING'}]} test_page = [{'f': [{'v': 'PI'}]}] test_output = gbq._parse_data(test_schema, test_page) - correct_output = DataFrame({'VALID_STRING': ['PI']}) + correct_output = DataFrame({'valid_string': ['PI']}) tm.assert_frame_equal(test_output, correct_output) def test_read_gbq_with_invalid_private_key_json_should_fail(self): @@ -438,12 +441,12 @@ def test_read_gbq_with_empty_private_key_file_should_fail(self): private_key=empty_file_path) def test_read_gbq_with_corrupted_private_key_json_should_fail(self): - _skip_if_no_private_key_path() + _skip_if_no_private_key_contents() with tm.assertRaises(gbq.InvalidPrivateKeyFormat): gbq.read_gbq( 'SELECT 1', project_id='x', - private_key=re.sub('[a-z]', '9', _get_private_key_path())) + private_key=re.sub('[a-z]', '9', _get_private_key_contents())) class TestReadGBQIntegration(tm.TestCase): @@ -478,112 +481,207 @@ def tearDown(self): pass def test_should_read_as_user_account(self): - if _in_travis_environment(): - pytest.skip("Cannot run local auth in travis environment") + _skip_local_auth_if_in_travis_env() - query = 'SELECT "PI" as VALID_STRING' + query = 'SELECT "PI" AS valid_string' df = gbq.read_gbq(query, project_id=_get_project_id()) - tm.assert_frame_equal(df, DataFrame({'VALID_STRING': ['PI']})) + tm.assert_frame_equal(df, DataFrame({'valid_string': ['PI']})) def test_should_read_as_service_account_with_key_path(self): _skip_if_no_private_key_path() - query = 'SELECT "PI" as VALID_STRING' + query = 'SELECT "PI" AS valid_string' df = gbq.read_gbq(query, project_id=_get_project_id(), private_key=_get_private_key_path()) - tm.assert_frame_equal(df, DataFrame({'VALID_STRING': ['PI']})) + tm.assert_frame_equal(df, DataFrame({'valid_string': ['PI']})) def test_should_read_as_service_account_with_key_contents(self): _skip_if_no_private_key_contents() - query = 'SELECT "PI" as VALID_STRING' + query = 'SELECT "PI" AS valid_string' df = gbq.read_gbq(query, project_id=_get_project_id(), private_key=_get_private_key_contents()) - tm.assert_frame_equal(df, DataFrame({'VALID_STRING': ['PI']})) + tm.assert_frame_equal(df, DataFrame({'valid_string': ['PI']})) + + +class TestReadGBQIntegrationWithServiceAccountKeyPath(tm.TestCase): + + @classmethod + def setUpClass(cls): + # - GLOBAL CLASS FIXTURES - + # put here any instruction you want to execute only *ONCE* *BEFORE* + # executing *ALL* tests described below. + + _skip_if_no_project_id() + _skip_if_no_private_key_path() + + _setup_common() + + def setUp(self): + # - PER-TEST FIXTURES - + # put here any instruction you want to be run *BEFORE* *EVERY* test is + # executed. + pass + + @classmethod + def tearDownClass(cls): + # - GLOBAL CLASS FIXTURES - + # put here any instruction you want to execute only *ONCE* *AFTER* + # executing all tests. + pass + + def tearDown(self): + # - PER-TEST FIXTURES - + # put here any instructions you want to be run *AFTER* *EVERY* test is + # executed. + pass def test_should_properly_handle_valid_strings(self): - query = 'SELECT "PI" as VALID_STRING' + query = 'SELECT "PI" AS valid_string' df = gbq.read_gbq(query, project_id=_get_project_id(), private_key=_get_private_key_path()) - tm.assert_frame_equal(df, DataFrame({'VALID_STRING': ['PI']})) + tm.assert_frame_equal(df, DataFrame({'valid_string': ['PI']})) def test_should_properly_handle_empty_strings(self): - query = 'SELECT "" as EMPTY_STRING' + query = 'SELECT "" AS empty_string' df = gbq.read_gbq(query, project_id=_get_project_id(), private_key=_get_private_key_path()) - tm.assert_frame_equal(df, DataFrame({'EMPTY_STRING': [""]})) + tm.assert_frame_equal(df, DataFrame({'empty_string': [""]})) def test_should_properly_handle_null_strings(self): - query = 'SELECT STRING(NULL) as NULL_STRING' + query = 'SELECT STRING(NULL) AS null_string' df = gbq.read_gbq(query, project_id=_get_project_id(), private_key=_get_private_key_path()) - tm.assert_frame_equal(df, DataFrame({'NULL_STRING': [None]})) + tm.assert_frame_equal(df, DataFrame({'null_string': [None]})) def test_should_properly_handle_valid_integers(self): - query = 'SELECT INTEGER(3) as VALID_INTEGER' + query = 'SELECT INTEGER(3) AS valid_integer' df = gbq.read_gbq(query, project_id=_get_project_id(), private_key=_get_private_key_path()) - tm.assert_frame_equal(df, DataFrame({'VALID_INTEGER': [3]})) + tm.assert_frame_equal(df, DataFrame({'valid_integer': [3]})) + + def test_should_properly_handle_nullable_integers(self): + query = '''SELECT * FROM + (SELECT 1 AS nullable_integer), + (SELECT NULL AS nullable_integer)''' + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) + tm.assert_frame_equal( + df, DataFrame({'nullable_integer': [1, None]}).astype(object)) + + def test_should_properly_handle_valid_longs(self): + query = 'SELECT 1 << 62 AS valid_long' + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) + tm.assert_frame_equal( + df, DataFrame({'valid_long': [1 << 62]})) + + def test_should_properly_handle_nullable_longs(self): + query = '''SELECT * FROM + (SELECT 1 << 62 AS nullable_long), + (SELECT NULL AS nullable_long)''' + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) + tm.assert_frame_equal( + df, DataFrame({'nullable_long': [1 << 62, None]}).astype(object)) def test_should_properly_handle_null_integers(self): - query = 'SELECT INTEGER(NULL) as NULL_INTEGER' + query = 'SELECT INTEGER(NULL) AS null_integer' df = gbq.read_gbq(query, project_id=_get_project_id(), private_key=_get_private_key_path()) - tm.assert_frame_equal(df, DataFrame({'NULL_INTEGER': [np.nan]})) + tm.assert_frame_equal(df, DataFrame({'null_integer': [None]})) def test_should_properly_handle_valid_floats(self): - query = 'SELECT PI() as VALID_FLOAT' + from math import pi + query = 'SELECT PI() AS valid_float' df = gbq.read_gbq(query, project_id=_get_project_id(), private_key=_get_private_key_path()) tm.assert_frame_equal(df, DataFrame( - {'VALID_FLOAT': [3.141592653589793]})) + {'valid_float': [pi]})) + + def test_should_properly_handle_nullable_floats(self): + from math import pi + query = '''SELECT * FROM + (SELECT PI() AS nullable_float), + (SELECT NULL AS nullable_float)''' + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) + tm.assert_frame_equal( + df, DataFrame({'nullable_float': [pi, None]})) + + def test_should_properly_handle_valid_doubles(self): + from math import pi + query = 'SELECT PI() * POW(10, 307) AS valid_double' + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) + tm.assert_frame_equal(df, DataFrame( + {'valid_double': [pi * 10 ** 307]})) + + def test_should_properly_handle_nullable_doubles(self): + from math import pi + query = '''SELECT * FROM + (SELECT PI() * POW(10, 307) AS nullable_double), + (SELECT NULL AS nullable_double)''' + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) + tm.assert_frame_equal( + df, DataFrame({'nullable_double': [pi * 10 ** 307, None]})) def test_should_properly_handle_null_floats(self): - query = 'SELECT FLOAT(NULL) as NULL_FLOAT' + query = 'SELECT FLOAT(NULL) AS null_float' df = gbq.read_gbq(query, project_id=_get_project_id(), private_key=_get_private_key_path()) - tm.assert_frame_equal(df, DataFrame({'NULL_FLOAT': [np.nan]})) + tm.assert_frame_equal(df, DataFrame({'null_float': [np.nan]})) def test_should_properly_handle_timestamp_unix_epoch(self): - query = 'SELECT TIMESTAMP("1970-01-01 00:00:00") as UNIX_EPOCH' + query = 'SELECT TIMESTAMP("1970-01-01 00:00:00") AS unix_epoch' df = gbq.read_gbq(query, project_id=_get_project_id(), private_key=_get_private_key_path()) tm.assert_frame_equal(df, DataFrame( - {'UNIX_EPOCH': [np.datetime64('1970-01-01T00:00:00.000000Z')]})) + {'unix_epoch': [np.datetime64('1970-01-01T00:00:00.000000Z')]})) def test_should_properly_handle_arbitrary_timestamp(self): - query = 'SELECT TIMESTAMP("2004-09-15 05:00:00") as VALID_TIMESTAMP' + query = 'SELECT TIMESTAMP("2004-09-15 05:00:00") AS valid_timestamp' df = gbq.read_gbq(query, project_id=_get_project_id(), private_key=_get_private_key_path()) tm.assert_frame_equal(df, DataFrame({ - 'VALID_TIMESTAMP': [np.datetime64('2004-09-15T05:00:00.000000Z')] + 'valid_timestamp': [np.datetime64('2004-09-15T05:00:00.000000Z')] })) def test_should_properly_handle_null_timestamp(self): - query = 'SELECT TIMESTAMP(NULL) as NULL_TIMESTAMP' + query = 'SELECT TIMESTAMP(NULL) AS null_timestamp' df = gbq.read_gbq(query, project_id=_get_project_id(), private_key=_get_private_key_path()) - tm.assert_frame_equal(df, DataFrame({'NULL_TIMESTAMP': [NaT]})) + tm.assert_frame_equal(df, DataFrame({'null_timestamp': [NaT]})) def test_should_properly_handle_true_boolean(self): - query = 'SELECT BOOLEAN(TRUE) as TRUE_BOOLEAN' + query = 'SELECT BOOLEAN(TRUE) AS true_boolean' df = gbq.read_gbq(query, project_id=_get_project_id(), private_key=_get_private_key_path()) - tm.assert_frame_equal(df, DataFrame({'TRUE_BOOLEAN': [True]})) + tm.assert_frame_equal(df, DataFrame({'true_boolean': [True]})) def test_should_properly_handle_false_boolean(self): - query = 'SELECT BOOLEAN(FALSE) as FALSE_BOOLEAN' + query = 'SELECT BOOLEAN(FALSE) AS false_boolean' df = gbq.read_gbq(query, project_id=_get_project_id(), private_key=_get_private_key_path()) - tm.assert_frame_equal(df, DataFrame({'FALSE_BOOLEAN': [False]})) + tm.assert_frame_equal(df, DataFrame({'false_boolean': [False]})) def test_should_properly_handle_null_boolean(self): - query = 'SELECT BOOLEAN(NULL) as NULL_BOOLEAN' + query = 'SELECT BOOLEAN(NULL) AS null_boolean' + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path()) + tm.assert_frame_equal(df, DataFrame({'null_boolean': [None]})) + + def test_should_properly_handle_nullable_booleans(self): + query = '''SELECT * FROM + (SELECT BOOLEAN(TRUE) AS nullable_boolean), + (SELECT NULL AS nullable_boolean)''' df = gbq.read_gbq(query, project_id=_get_project_id(), private_key=_get_private_key_path()) - tm.assert_frame_equal(df, DataFrame({'NULL_BOOLEAN': [None]})) + tm.assert_frame_equal( + df, DataFrame({'nullable_boolean': [True, None]}).astype(object)) def test_unicode_string_conversion_and_normalization(self): correct_test_datatype = DataFrame( - {'UNICODE_STRING': [u("\xe9\xfc")]} + {'unicode_string': [u("\xe9\xfc")]} ) unicode_string = "\xc3\xa9\xc3\xbc" @@ -591,40 +689,40 @@ def test_unicode_string_conversion_and_normalization(self): if compat.PY3: unicode_string = unicode_string.encode('latin-1').decode('utf8') - query = 'SELECT "{0}" as UNICODE_STRING'.format(unicode_string) + query = 'SELECT "{0}" AS unicode_string'.format(unicode_string) df = gbq.read_gbq(query, project_id=_get_project_id(), private_key=_get_private_key_path()) tm.assert_frame_equal(df, correct_test_datatype) def test_index_column(self): - query = "SELECT 'a' as STRING_1, 'b' as STRING_2" + query = "SELECT 'a' AS string_1, 'b' AS string_2" result_frame = gbq.read_gbq(query, project_id=_get_project_id(), - index_col="STRING_1", + index_col="string_1", private_key=_get_private_key_path()) correct_frame = DataFrame( - {'STRING_1': ['a'], 'STRING_2': ['b']}).set_index("STRING_1") + {'string_1': ['a'], 'string_2': ['b']}).set_index("string_1") tm.assert_equal(result_frame.index.name, correct_frame.index.name) def test_column_order(self): - query = "SELECT 'a' as STRING_1, 'b' as STRING_2, 'c' as STRING_3" - col_order = ['STRING_3', 'STRING_1', 'STRING_2'] + query = "SELECT 'a' AS string_1, 'b' AS string_2, 'c' AS string_3" + col_order = ['string_3', 'string_1', 'string_2'] result_frame = gbq.read_gbq(query, project_id=_get_project_id(), col_order=col_order, private_key=_get_private_key_path()) - correct_frame = DataFrame({'STRING_1': ['a'], 'STRING_2': [ - 'b'], 'STRING_3': ['c']})[col_order] + correct_frame = DataFrame({'string_1': ['a'], 'string_2': [ + 'b'], 'string_3': ['c']})[col_order] tm.assert_frame_equal(result_frame, correct_frame) def test_column_order_plus_index(self): - query = "SELECT 'a' as STRING_1, 'b' as STRING_2, 'c' as STRING_3" - col_order = ['STRING_3', 'STRING_2'] + query = "SELECT 'a' AS string_1, 'b' AS string_2, 'c' AS string_3" + col_order = ['string_3', 'string_2'] result_frame = gbq.read_gbq(query, project_id=_get_project_id(), - index_col='STRING_1', col_order=col_order, + index_col='string_1', col_order=col_order, private_key=_get_private_key_path()) correct_frame = DataFrame( - {'STRING_1': ['a'], 'STRING_2': ['b'], 'STRING_3': ['c']}) - correct_frame.set_index('STRING_1', inplace=True) + {'string_1': ['a'], 'string_2': ['b'], 'string_3': ['c']}) + correct_frame.set_index('string_1', inplace=True) correct_frame = correct_frame[col_order] tm.assert_frame_equal(result_frame, correct_frame) @@ -658,14 +756,17 @@ def test_download_dataset_larger_than_200k_rows(self): def test_zero_rows(self): # Bug fix for https://github.com/pandas-dev/pandas/issues/10273 - df = gbq.read_gbq("SELECT title, id " + df = gbq.read_gbq("SELECT title, id, is_bot, " + "SEC_TO_TIMESTAMP(timestamp) ts " "FROM [publicdata:samples.wikipedia] " "WHERE timestamp=-9999999", project_id=_get_project_id(), private_key=_get_private_key_path()) page_array = np.zeros( - (0,), dtype=[('title', object), ('id', np.dtype(float))]) - expected_result = DataFrame(page_array, columns=['title', 'id']) + (0,), dtype=[('title', object), ('id', np.dtype(int)), + ('is_bot', np.dtype(bool)), ('ts', 'M8[ns]')]) + expected_result = DataFrame( + page_array, columns=['title', 'id', 'is_bot', 'ts']) self.assert_frame_equal(df, expected_result) def test_legacy_sql(self): @@ -718,7 +819,7 @@ def test_invalid_option_for_sql_dialect(self): dialect='standard', private_key=_get_private_key_path()) def test_query_with_parameters(self): - sql_statement = "SELECT @param1 + @param2 as VALID_RESULT" + sql_statement = "SELECT @param1 + @param2 AS valid_result" config = { 'query': { "useLegacySql": False, @@ -756,11 +857,11 @@ def test_query_with_parameters(self): df = gbq.read_gbq(sql_statement, project_id=_get_project_id(), private_key=_get_private_key_path(), configuration=config) - tm.assert_frame_equal(df, DataFrame({'VALID_RESULT': [3]})) + tm.assert_frame_equal(df, DataFrame({'valid_result': [3]})) def test_query_inside_configuration(self): - query_no_use = 'SELECT "PI_WRONG" as VALID_STRING' - query = 'SELECT "PI" as VALID_STRING' + query_no_use = 'SELECT "PI_WRONG" AS valid_string' + query = 'SELECT "PI" AS valid_string' config = { 'query': { "query": query, @@ -777,7 +878,7 @@ def test_query_inside_configuration(self): df = gbq.read_gbq(None, project_id=_get_project_id(), private_key=_get_private_key_path(), configuration=config) - tm.assert_frame_equal(df, DataFrame({'VALID_STRING': ['PI']})) + tm.assert_frame_equal(df, DataFrame({'valid_string': ['PI']})) def test_configuration_without_query(self): sql_statement = 'SELECT 1' @@ -803,7 +904,7 @@ def test_configuration_without_query(self): configuration=config) -class TestToGBQIntegration(tm.TestCase): +class TestToGBQIntegrationWithServiceAccountKeyPath(tm.TestCase): # Changes to BigQuery table schema may take up to 2 minutes as of May 2015 # As a workaround to this issue, each test should use a unique table name. # Make sure to modify the for loop range in the tearDownClass when a new @@ -817,6 +918,7 @@ def setUpClass(cls): # executing *ALL* tests described below. _skip_if_no_project_id() + _skip_if_no_private_key_path() _setup_common() @@ -860,11 +962,11 @@ def test_upload_data(self): sleep(30) # <- Curses Google!!! - result = gbq.read_gbq("SELECT COUNT(*) as NUM_ROWS FROM {0}" + result = gbq.read_gbq("SELECT COUNT(*) AS num_rows FROM {0}" .format(self.destination_table + test_id), project_id=_get_project_id(), private_key=_get_private_key_path()) - self.assertEqual(result['NUM_ROWS'][0], test_size) + self.assertEqual(result['num_rows'][0], test_size) def test_upload_data_if_table_exists_fail(self): test_id = "2" @@ -898,11 +1000,11 @@ def test_upload_data_if_table_exists_append(self): sleep(30) # <- Curses Google!!! - result = gbq.read_gbq("SELECT COUNT(*) as NUM_ROWS FROM {0}" + result = gbq.read_gbq("SELECT COUNT(*) AS num_rows FROM {0}" .format(self.destination_table + test_id), project_id=_get_project_id(), private_key=_get_private_key_path()) - self.assertEqual(result['NUM_ROWS'][0], test_size * 2) + self.assertEqual(result['num_rows'][0], test_size * 2) # Try inserting with a different schema, confirm failure with tm.assertRaises(gbq.InvalidSchema): @@ -927,11 +1029,11 @@ def test_upload_data_if_table_exists_replace(self): sleep(30) # <- Curses Google!!! - result = gbq.read_gbq("SELECT COUNT(*) as NUM_ROWS FROM {0}" + result = gbq.read_gbq("SELECT COUNT(*) AS num_rows FROM {0}" .format(self.destination_table + test_id), project_id=_get_project_id(), private_key=_get_private_key_path()) - self.assertEqual(result['NUM_ROWS'][0], 5) + self.assertEqual(result['num_rows'][0], 5) def test_google_upload_errors_should_raise_exception(self): raise pytest.skip("buggy test") @@ -1110,7 +1212,7 @@ def test_dataset_does_not_exist(self): 'Expected dataset not to exist') -class TestToGBQIntegrationServiceAccountKeyPath(tm.TestCase): +class TestToGBQIntegrationWithLocalUserAccountAuth(tm.TestCase): # Changes to BigQuery table schema may take up to 2 minutes as of May 2015 # As a workaround to this issue, each test should use a unique table name. # Make sure to modify the for loop range in the tearDownClass when a new @@ -1125,7 +1227,7 @@ def setUpClass(cls): # executing *ALL* tests described below. _skip_if_no_project_id() - _skip_if_no_private_key_path() + _skip_local_auth_if_in_travis_env() _setup_common() @@ -1135,7 +1237,7 @@ def setUp(self): # is executed. self.dataset_prefix = _get_dataset_prefix_random() - clean_gbq_environment(self.dataset_prefix, _get_private_key_path()) + clean_gbq_environment(self.dataset_prefix) self.destination_table = "{0}{1}.{2}".format(self.dataset_prefix, "2", TABLE_ID) @@ -1152,25 +1254,24 @@ def tearDown(self): # is executed. clean_gbq_environment(self.dataset_prefix, _get_private_key_path()) - def test_upload_data_as_service_account_with_key_path(self): + def test_upload_data(self): test_id = "1" test_size = 10 df = make_mixed_dataframe_v2(test_size) gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), - chunksize=10000, private_key=_get_private_key_path()) + chunksize=10000) sleep(30) # <- Curses Google!!! - result = gbq.read_gbq("SELECT COUNT(*) as NUM_ROWS FROM {0}".format( + result = gbq.read_gbq("SELECT COUNT(*) AS num_rows FROM {0}".format( self.destination_table + test_id), - project_id=_get_project_id(), - private_key=_get_private_key_path()) + project_id=_get_project_id()) - self.assertEqual(result['NUM_ROWS'][0], test_size) + self.assertEqual(result['num_rows'][0], test_size) -class TestToGBQIntegrationServiceAccountKeyContents(tm.TestCase): +class TestToGBQIntegrationWithServiceAccountKeyContents(tm.TestCase): # Changes to BigQuery table schema may take up to 2 minutes as of May 2015 # As a workaround to this issue, each test should use a unique table name. # Make sure to modify the for loop range in the tearDownClass when a new @@ -1211,7 +1312,7 @@ def tearDown(self): # is executed. clean_gbq_environment(self.dataset_prefix, _get_private_key_contents()) - def test_upload_data_as_service_account_with_key_contents(self): + def test_upload_data(self): test_id = "1" test_size = 10 df = make_mixed_dataframe_v2(test_size) @@ -1221,8 +1322,8 @@ def test_upload_data_as_service_account_with_key_contents(self): sleep(30) # <- Curses Google!!! - result = gbq.read_gbq("SELECT COUNT(*) as NUM_ROWS FROM {0}".format( + result = gbq.read_gbq("SELECT COUNT(*) as num_rows FROM {0}".format( self.destination_table + test_id), project_id=_get_project_id(), private_key=_get_private_key_contents()) - self.assertEqual(result['NUM_ROWS'][0], test_size) + self.assertEqual(result['num_rows'][0], test_size) From b58670416f3b36523e38984c86ef16058806cfea Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 15 Feb 2017 10:48:05 -0500 Subject: [PATCH 019/519] update README.rst with build status --- packages/pandas-gbq/README.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/packages/pandas-gbq/README.rst b/packages/pandas-gbq/README.rst index b83c15bb97ee..c191db7cf584 100644 --- a/packages/pandas-gbq/README.rst +++ b/packages/pandas-gbq/README.rst @@ -1,3 +1,22 @@ + + + + + + + + + + + + + +
Latest Releaselatest release
Package Statusstatus
Build Status + + travis build status + +
+ pandas-gbq ========== From bb29089d90d3807508d3ac1a1f18406f4f0ef4eb Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 15 Feb 2017 10:50:30 -0500 Subject: [PATCH 020/519] update README.rst in .rst format! --- packages/pandas-gbq/README.rst | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/packages/pandas-gbq/README.rst b/packages/pandas-gbq/README.rst index c191db7cf584..cf798f85fc80 100644 --- a/packages/pandas-gbq/README.rst +++ b/packages/pandas-gbq/README.rst @@ -1,21 +1,7 @@ - - - - - - - - - - - - - -
Latest Releaselatest release
Package Statusstatus
Build Status - - travis build status - -
+.. |Build Status| image:: https://travis-ci.org/pydata/pandas-gbq.svg?branch=master + :target: https://travis-ci.org/pydata/pandas-gbq +.. |Version Status| image:: https://img.shields.io/pypi/v/pandas-gbq.svg + :target: https://pypi.python.org/pypi/pandas-gbq/ pandas-gbq ========== From 3dade544652096d5dcb8505a9273b81f14b8164a Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 15 Feb 2017 10:51:21 -0500 Subject: [PATCH 021/519] try again --- packages/pandas-gbq/README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/pandas-gbq/README.rst b/packages/pandas-gbq/README.rst index cf798f85fc80..2cd43725f13e 100644 --- a/packages/pandas-gbq/README.rst +++ b/packages/pandas-gbq/README.rst @@ -6,6 +6,8 @@ pandas-gbq ========== +|Build Status| |Version Status| + **pandas-gbq** is a package providing an interface to the Google Big Query interface from pandas From a4d74516819c27eb5a443c8a4415edc8866b4e2c Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Sun, 19 Feb 2017 12:35:58 -0500 Subject: [PATCH 022/519] Remove version added references in docstrings (#11) --- packages/pandas-gbq/pandas_gbq/gbq.py | 28 --------------------------- 1 file changed, 28 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 64985dbd67a1..9759e37932b8 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -167,8 +167,6 @@ def get_application_default_credentials(self): This method tries to retrieve the "default application credentials". This could be useful for running code on Google Cloud Platform. - .. versionadded:: 0.19.0 - Parameters ---------- None @@ -652,8 +650,6 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, By default "application default credentials" are used. - .. versionadded:: 0.19.0 - If default application credentials are not found or are restrictive, user account credentials are used. In this case, you will be asked to grant permissions for product name 'pandas GBQ'. @@ -683,8 +679,6 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, or string contents. This is useful for remote server authentication (eg. jupyter iPython notebook on remote host) - .. versionadded:: 0.18.1 - dialect : {'legacy', 'standard'}, default 'legacy' 'legacy' : Use BigQuery's legacy SQL dialect. 'standard' : Use BigQuery's standard SQL (beta), which is @@ -692,8 +686,6 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, see `BigQuery SQL Reference `__ - .. versionadded:: 0.19.0 - **kwargs : Arbitrary keyword arguments configuration (dict): query config parameters for job processing. For example: @@ -703,8 +695,6 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, For more information see `BigQuery SQL Reference ` - .. versionadded:: 0.20.0 - Returns ------- df: DataFrame @@ -788,8 +778,6 @@ def to_gbq(dataframe, destination_table, project_id, chunksize=10000, By default "application default credentials" are used. - .. versionadded:: 0.19.0 - If default application credentials are not found or are restrictive, user account credentials are used. In this case, you will be asked to grant permissions for product name 'pandas GBQ'. @@ -912,8 +900,6 @@ def __init__(self, project_id, dataset_id, reauth=False, verbose=False, def exists(self, table_id): """ Check if a table exists in Google BigQuery - .. versionadded:: 0.17.0 - Parameters ---------- table : str @@ -940,8 +926,6 @@ def exists(self, table_id): def create(self, table_id, schema): """ Create a table in Google BigQuery given a table and schema - .. versionadded:: 0.17.0 - Parameters ---------- table : str @@ -980,8 +964,6 @@ def create(self, table_id, schema): def delete(self, table_id): """ Delete a table in Google BigQuery - .. versionadded:: 0.17.0 - Parameters ---------- table : str @@ -1017,8 +999,6 @@ def __init__(self, project_id, reauth=False, verbose=False, def exists(self, dataset_id): """ Check if a dataset exists in Google BigQuery - .. versionadded:: 0.17.0 - Parameters ---------- dataset_id : str @@ -1044,8 +1024,6 @@ def exists(self, dataset_id): def datasets(self): """ Return a list of datasets in Google BigQuery - .. versionadded:: 0.17.0 - Parameters ---------- None @@ -1086,8 +1064,6 @@ def datasets(self): def create(self, dataset_id): """ Create a dataset in Google BigQuery - .. versionadded:: 0.17.0 - Parameters ---------- dataset : str @@ -1115,8 +1091,6 @@ def create(self, dataset_id): def delete(self, dataset_id): """ Delete a dataset in Google BigQuery - .. versionadded:: 0.17.0 - Parameters ---------- dataset : str @@ -1140,8 +1114,6 @@ def delete(self, dataset_id): def tables(self, dataset_id): """ List tables in the specific dataset in Google BigQuery - .. versionadded:: 0.17.0 - Parameters ---------- dataset : str From c0979702c76ac6377cd87f06a8ad362a427d7a6f Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 22 Feb 2017 16:01:34 -0500 Subject: [PATCH 023/519] add in release-procedure.md --- packages/pandas-gbq/pandas_gbq/__init__.py | 6 ++++++ packages/pandas-gbq/release-procedure.md | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 packages/pandas-gbq/release-procedure.md diff --git a/packages/pandas-gbq/pandas_gbq/__init__.py b/packages/pandas-gbq/pandas_gbq/__init__.py index ad1658ec0c09..40268bc31ea3 100644 --- a/packages/pandas-gbq/pandas_gbq/__init__.py +++ b/packages/pandas-gbq/pandas_gbq/__init__.py @@ -1 +1,7 @@ from .gbq import to_gbq, read_gbq # noqa + +# use the closest tagged version if possible +from ._version import get_versions +v = get_versions() +__version__ = v.get('closest-tag', v['version']) +del get_versions, v diff --git a/packages/pandas-gbq/release-procedure.md b/packages/pandas-gbq/release-procedure.md new file mode 100644 index 000000000000..e6adee983f08 --- /dev/null +++ b/packages/pandas-gbq/release-procedure.md @@ -0,0 +1,19 @@ +* Tag commit + + git tag -a x.x.x -m 'Version x.x.x' + +* and push to github + + git push pandas-gbq master --tags + +* Upload to PyPI + + git clean -xfd + python setup.py register sdist bdist_wheel --universal + twine upload dist/* + +* Update anaconda recipe. + + This should happen automatically within a day or two. + +* Update conda recipe feedstock on `conda-forge `_. From 46731d110d96167b6a79e98e5da20a4e84d4ac48 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 22 Feb 2017 16:06:43 -0500 Subject: [PATCH 024/519] update versioneer --- packages/pandas-gbq/.gitattributes | 1 + packages/pandas-gbq/MANIFEST.in | 2 + packages/pandas-gbq/pandas_gbq/__init__.py | 12 +- packages/pandas-gbq/pandas_gbq/_version.py | 520 +++++++++++++++ packages/pandas-gbq/versioneer.py | 739 ++++++++++++--------- 5 files changed, 962 insertions(+), 312 deletions(-) create mode 100644 packages/pandas-gbq/.gitattributes create mode 100644 packages/pandas-gbq/MANIFEST.in create mode 100644 packages/pandas-gbq/pandas_gbq/_version.py diff --git a/packages/pandas-gbq/.gitattributes b/packages/pandas-gbq/.gitattributes new file mode 100644 index 000000000000..9afa48862e72 --- /dev/null +++ b/packages/pandas-gbq/.gitattributes @@ -0,0 +1 @@ +pandas_gbq/_version.py export-subst diff --git a/packages/pandas-gbq/MANIFEST.in b/packages/pandas-gbq/MANIFEST.in new file mode 100644 index 000000000000..ec2ec72a9a4f --- /dev/null +++ b/packages/pandas-gbq/MANIFEST.in @@ -0,0 +1,2 @@ +include versioneer.py +include pandas_gbq/_version.py diff --git a/packages/pandas-gbq/pandas_gbq/__init__.py b/packages/pandas-gbq/pandas_gbq/__init__.py index 40268bc31ea3..fc9cab62fd60 100644 --- a/packages/pandas-gbq/pandas_gbq/__init__.py +++ b/packages/pandas-gbq/pandas_gbq/__init__.py @@ -1,7 +1,11 @@ from .gbq import to_gbq, read_gbq # noqa -# use the closest tagged version if possible from ._version import get_versions -v = get_versions() -__version__ = v.get('closest-tag', v['version']) -del get_versions, v +versions = get_versions() +__version__ = versions['version'] +__git_revision__ = versions['full-revisionid'] +del get_versions, versions + +from ._version import get_versions +__version__ = get_versions()['version'] +del get_versions diff --git a/packages/pandas-gbq/pandas_gbq/_version.py b/packages/pandas-gbq/pandas_gbq/_version.py new file mode 100644 index 000000000000..6f81e4ccfade --- /dev/null +++ b/packages/pandas-gbq/pandas_gbq/_version.py @@ -0,0 +1,520 @@ + +# This file helps to compute a version number in source trees obtained from +# git-archive tarball (such as those provided by githubs download-from-tag +# feature). Distribution tarballs (built by setup.py sdist) and build +# directories (produced by setup.py build) will contain a much shorter file +# that just contains the computed version number. + +# This file is released into the public domain. Generated by +# versioneer-0.18 (https://github.com/warner/python-versioneer) + +"""Git implementation of _version.py.""" + +import errno +import os +import re +import subprocess +import sys + + +def get_keywords(): + """Get the keywords needed to look up the version information.""" + # these strings will be replaced by git during git-archive. + # setup.py/versioneer.py will grep for the variable names, so they must + # each be defined on a line of their own. _version.py will just call + # get_keywords(). + git_refnames = "$Format:%d$" + git_full = "$Format:%H$" + git_date = "$Format:%ci$" + keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} + return keywords + + +class VersioneerConfig: + """Container for Versioneer configuration parameters.""" + + +def get_config(): + """Create, populate and return the VersioneerConfig() object.""" + # these strings are filled in when 'setup.py versioneer' creates + # _version.py + cfg = VersioneerConfig() + cfg.VCS = "git" + cfg.style = "pep440" + cfg.tag_prefix = "v" + cfg.parentdir_prefix = "pandas_gbq-" + cfg.versionfile_source = "pandas_gbq/_version.py" + cfg.verbose = False + return cfg + + +class NotThisMethod(Exception): + """Exception raised if a method is not valid for the current scenario.""" + + +LONG_VERSION_PY = {} +HANDLERS = {} + + +def register_vcs_handler(vcs, method): # decorator + """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate + + +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, + env=None): + """Call the given command(s).""" + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen([c] + args, cwd=cwd, env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) + break + except EnvironmentError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %s" % dispcmd) + print(e) + return None, None + else: + if verbose: + print("unable to find command, tried %s" % (commands,)) + return None, None + stdout = p.communicate()[0].strip() + if sys.version_info[0] >= 3: + stdout = stdout.decode() + if p.returncode != 0: + if verbose: + print("unable to run %s (error)" % dispcmd) + print("stdout was %s" % stdout) + return None, p.returncode + return stdout, p.returncode + + +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. + + Source tarballs conventionally unpack into a directory that includes both + the project name and a version string. We will also support searching up + two directory levels for an appropriately named parent directory + """ + rootdirs = [] + + for i in range(3): + dirname = os.path.basename(root) + if dirname.startswith(parentdir_prefix): + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None, "date": None} + else: + rootdirs.append(root) + root = os.path.dirname(root) # up a level + + if verbose: + print("Tried directories %s but none started with prefix %s" % + (str(rootdirs), parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + + +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" + # the code embedded in _version.py can just fetch the value of these + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} + try: + f = open(versionfile_abs, "r") + for line in f.readlines(): + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) + f.close() + except EnvironmentError: + pass + return keywords + + +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + date = keywords.get("date") + if date is not None: + # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant + # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 + # -like" string, which we must then edit to make compliant), because + # it's been around since git-1.5.3, and it's too difficult to + # discover which version we're using, or to work around using an + # older one. + date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + refnames = keywords["refnames"].strip() + if refnames.startswith("$Format"): + if verbose: + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") + refs = set([r.strip() for r in refnames.strip("()").split(",")]) + # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of + # just "foo-1.0". If we see a "tag: " prefix, prefer those. + TAG = "tag: " + tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + if not tags: + # Either we're using git < 1.8.3, or there really are no tags. We use + # a heuristic: assume all version tags have a digit. The old git %d + # expansion behaves like git log --decorate=short and strips out the + # refs/heads/ and refs/tags/ prefixes that would let us distinguish + # between branches and tags. By ignoring refnames without digits, we + # filter out many common branch names like "release" and + # "stabilization", as well as "HEAD" and "master". + tags = set([r for r in refs if re.search(r'\d', r)]) + if verbose: + print("discarding '%s', no digits" % ",".join(refs - tags)) + if verbose: + print("likely tags: %s" % ",".join(sorted(tags))) + for ref in sorted(tags): + # sorting will prefer e.g. "2.0" over "2.0rc1" + if ref.startswith(tag_prefix): + r = ref[len(tag_prefix):] + if verbose: + print("picking %s" % r) + return {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None, + "date": date} + # no suitable tags, so version is "0+unknown", but full hex is still there + if verbose: + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags", "date": None} + + +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] + + out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, + hide_stderr=True) + if rc != 0: + if verbose: + print("Directory %s not under git control" % root) + raise NotThisMethod("'git rev-parse --git-dir' returned error") + + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%s*" % tag_prefix], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%s'" + % describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%s' doesn't start with prefix '%s'" + print(fmt % (full_tag, tag_prefix)) + pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" + % (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + + else: + # HEX: no tags + pieces["closest-tag"] = None + count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits + + # commit date: see ISO-8601 comment in git_versions_from_keywords() + date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], + cwd=root)[0].strip() + pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + + return pieces + + +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" + + +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". + + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + """TAG[.post.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post.dev%d" % pieces["distance"] + else: + # exception #1 + rendered = "0.post.dev%d" % pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + return rendered + + +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + + The ".dev0" means dirty. + + Eexceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered + + +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. + + Like 'git describe --tags --dirty --always'. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. + + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None} + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%s'" % style) + + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None, + "date": pieces.get("date")} + + +def get_versions(): + """Get version information or return default if unable to do so.""" + # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have + # __file__, we can work backwards from there to the root. Some + # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which + # case we can only use expanded keywords. + + cfg = get_config() + verbose = cfg.verbose + + try: + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, + verbose) + except NotThisMethod: + pass + + try: + root = os.path.realpath(__file__) + # versionfile_source is the relative path from the top of the source + # tree (where the .git directory might live) to this file. Invert + # this to find the root from __file__. + for i in cfg.versionfile_source.split('/'): + root = os.path.dirname(root) + except NameError: + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree", + "date": None} + + try: + pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) + return render(pieces, cfg.style) + except NotThisMethod: + pass + + try: + if cfg.parentdir_prefix: + return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + except NotThisMethod: + pass + + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to compute version", "date": None} diff --git a/packages/pandas-gbq/versioneer.py b/packages/pandas-gbq/versioneer.py index c010f63e3ead..64fea1c89272 100644 --- a/packages/pandas-gbq/versioneer.py +++ b/packages/pandas-gbq/versioneer.py @@ -1,7 +1,8 @@ -# Version: 0.15 +# Version: 0.18 + +"""The Versioneer - like a rocketeer, but for versions. -""" The Versioneer ============== @@ -9,7 +10,7 @@ * https://github.com/warner/python-versioneer * Brian Warner * License: Public Domain -* Compatible With: python2.6, 2.7, 3.2, 3.3, 3.4, and pypy +* Compatible With: python2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, and pypy * [![Latest Version] (https://pypip.in/version/versioneer/badge.svg?style=flat) ](https://pypi.python.org/pypi/versioneer/) @@ -87,125 +88,7 @@ ## Installation -First, decide on values for the following configuration variables: - -* `VCS`: the version control system you use. Currently accepts "git". - -* `style`: the style of version string to be produced. See "Styles" below for - details. Defaults to "pep440", which looks like - `TAG[+DISTANCE.gSHORTHASH[.dirty]]`. - -* `versionfile_source`: - - A project-relative pathname into which the generated version strings should - be written. This is usually a `_version.py` next to your project's main - `__init__.py` file, so it can be imported at runtime. If your project uses - `src/myproject/__init__.py`, this should be `src/myproject/_version.py`. - This file should be checked in to your VCS as usual: the copy created below - by `setup.py setup_versioneer` will include code that parses expanded VCS - keywords in generated tarballs. The 'build' and 'sdist' commands will - replace it with a copy that has just the calculated version string. - - This must be set even if your project does not have any modules (and will - therefore never import `_version.py`), since "setup.py sdist" -based trees - still need somewhere to record the pre-calculated version strings. Anywhere - in the source tree should do. If there is a `__init__.py` next to your - `_version.py`, the `setup.py setup_versioneer` command (described below) - will append some `__version__`-setting assignments, if they aren't already - present. - -* `versionfile_build`: - - Like `versionfile_source`, but relative to the build directory instead of - the source directory. These will differ when your setup.py uses - 'package_dir='. If you have `package_dir={'myproject': 'src/myproject'}`, - then you will probably have `versionfile_build='myproject/_version.py'` and - `versionfile_source='src/myproject/_version.py'`. - - If this is set to None, then `setup.py build` will not attempt to rewrite - any `_version.py` in the built tree. If your project does not have any - libraries (e.g. if it only builds a script), then you should use - `versionfile_build = None` and override `distutils.command.build_scripts` - to explicitly insert a copy of `versioneer.get_version()` into your - generated script. - -* `tag_prefix`: - - a string, like 'PROJECTNAME-', which appears at the start of all VCS tags. - If your tags look like 'myproject-1.2.0', then you should use - tag_prefix='myproject-'. If you use unprefixed tags like '1.2.0', this - should be an empty string. - -* `parentdir_prefix`: - - a optional string, frequently the same as tag_prefix, which appears at the - start of all unpacked tarball filenames. If your tarball unpacks into - 'myproject-1.2.0', this should be 'myproject-'. To disable this feature, - just omit the field from your `setup.cfg`. - -This tool provides one script, named `versioneer`. That script has one mode, -"install", which writes a copy of `versioneer.py` into the current directory -and runs `versioneer.py setup` to finish the installation. - -To versioneer-enable your project: - -* 1: Modify your `setup.cfg`, adding a section named `[versioneer]` and - populating it with the configuration values you decided earlier (note that - the option names are not case-sensitive): - - ```` - [versioneer] - VCS = git - style = pep440 - versionfile_source = src/myproject/_version.py - versionfile_build = myproject/_version.py - tag_prefix = "" - parentdir_prefix = myproject- - ```` - -* 2: Run `versioneer install`. This will do the following: - - * copy `versioneer.py` into the top of your source tree - * create `_version.py` in the right place (`versionfile_source`) - * modify your `__init__.py` (if one exists next to `_version.py`) to define - `__version__` (by calling a function from `_version.py`) - * modify your `MANIFEST.in` to include both `versioneer.py` and the - generated `_version.py` in sdist tarballs - - `versioneer install` will complain about any problems it finds with your - `setup.py` or `setup.cfg`. Run it multiple times until you have fixed all - the problems. - -* 3: add a `import versioneer` to your setup.py, and add the following - arguments to the setup() call: - - version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), - -* 4: commit these changes to your VCS. To make sure you won't forget, - `versioneer install` will mark everything it touched for addition using - `git add`. Don't forget to add `setup.py` and `setup.cfg` too. - -## Post-Installation Usage - -Once established, all uses of your tree from a VCS checkout should get the -current version string. All generated tarballs should include an embedded -version string (so users who unpack them will not need a VCS tool installed). - -If you distribute your project through PyPI, then the release process should -boil down to two steps: - -* 1: git tag 1.0 -* 2: python setup.py register sdist upload - -If you distribute it through github (i.e. users use github to generate -tarballs with `git archive`), the process is: - -* 1: git tag 1.0 -* 2: git push; git push --tags - -Versioneer will report "0+untagged.NUMCOMMITS.gHASH" until your tree has at -least one tag in its history. +See [INSTALL.md](./INSTALL.md) for detailed installation instructions. ## Version-String Flavors @@ -226,6 +109,10 @@ * `['full-revisionid']`: detailed revision identifier. For Git, this is the full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac". +* `['date']`: Date and time of the latest `HEAD` commit. For Git, it is the + commit date in ISO 8601 format. This will be None if the date is not + available. + * `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that this is only accurate if run in a VCS checkout, otherwise it is likely to be False or None @@ -264,8 +151,8 @@ software (exactly equal to a known tag), the identifier will only contain the stripped tag, e.g. "0.11". -Other styles are available. See details.md in the Versioneer source tree for -descriptions. +Other styles are available. See [details.md](details.md) in the Versioneer +source tree for descriptions. ## Debugging @@ -275,47 +162,95 @@ display the full contents of `get_versions()` (including the `error` string, which may help identify what went wrong). -## Updating Versioneer +## Known Limitations -To upgrade your project to a new release of Versioneer, do the following: +Some situations are known to cause problems for Versioneer. This details the +most significant ones. More can be found on Github +[issues page](https://github.com/warner/python-versioneer/issues). -* install the new Versioneer (`pip install -U versioneer` or equivalent) -* edit `setup.cfg`, if necessary, to include any new configuration settings - indicated by the release notes -* re-run `versioneer install` in your source tree, to replace - `SRC/_version.py` -* commit any changed files +### Subprojects + +Versioneer has limited support for source trees in which `setup.py` is not in +the root directory (e.g. `setup.py` and `.git/` are *not* siblings). The are +two common reasons why `setup.py` might not be in the root: + +* Source trees which contain multiple subprojects, such as + [Buildbot](https://github.com/buildbot/buildbot), which contains both + "master" and "slave" subprojects, each with their own `setup.py`, + `setup.cfg`, and `tox.ini`. Projects like these produce multiple PyPI + distributions (and upload multiple independently-installable tarballs). +* Source trees whose main purpose is to contain a C library, but which also + provide bindings to Python (and perhaps other langauges) in subdirectories. + +Versioneer will look for `.git` in parent directories, and most operations +should get the right version string. However `pip` and `setuptools` have bugs +and implementation details which frequently cause `pip install .` from a +subproject directory to fail to find a correct version string (so it usually +defaults to `0+unknown`). + +`pip install --editable .` should work correctly. `setup.py install` might +work too. + +Pip-8.1.1 is known to have this problem, but hopefully it will get fixed in +some later version. + +[Bug #38](https://github.com/warner/python-versioneer/issues/38) is tracking +this issue. The discussion in +[PR #61](https://github.com/warner/python-versioneer/pull/61) describes the +issue from the Versioneer side in more detail. +[pip PR#3176](https://github.com/pypa/pip/pull/3176) and +[pip PR#3615](https://github.com/pypa/pip/pull/3615) contain work to improve +pip to let Versioneer work correctly. + +Versioneer-0.16 and earlier only looked for a `.git` directory next to the +`setup.cfg`, so subprojects were completely unsupported with those releases. -### Upgrading to 0.15 +### Editable installs with setuptools <= 18.5 -Starting with this version, Versioneer is configured with a `[versioneer]` -section in your `setup.cfg` file. Earlier versions required the `setup.py` to -set attributes on the `versioneer` module immediately after import. The new -version will refuse to run (raising an exception during import) until you -have provided the necessary `setup.cfg` section. +`setup.py develop` and `pip install --editable .` allow you to install a +project into a virtualenv once, then continue editing the source code (and +test) without re-installing after every change. -In addition, the Versioneer package provides an executable named -`versioneer`, and the installation process is driven by running `versioneer -install`. In 0.14 and earlier, the executable was named -`versioneer-installer` and was run without an argument. +"Entry-point scripts" (`setup(entry_points={"console_scripts": ..})`) are a +convenient way to specify executable scripts that should be installed along +with the python package. -### Upgrading to 0.14 +These both work as expected when using modern setuptools. When using +setuptools-18.5 or earlier, however, certain operations will cause +`pkg_resources.DistributionNotFound` errors when running the entrypoint +script, which must be resolved by re-installing the package. This happens +when the install happens with one version, then the egg_info data is +regenerated while a different version is checked out. Many setup.py commands +cause egg_info to be rebuilt (including `sdist`, `wheel`, and installing into +a different virtualenv), so this can be surprising. -0.14 changes the format of the version string. 0.13 and earlier used -hyphen-separated strings like "0.11-2-g1076c97-dirty". 0.14 and beyond use a -plus-separated "local version" section strings, with dot-separated -components, like "0.11+2.g1076c97". PEP440-strict tools did not like the old -format, but should be ok with the new one. +[Bug #83](https://github.com/warner/python-versioneer/issues/83) describes +this one, but upgrading to a newer version of setuptools should probably +resolve it. -### Upgrading from 0.11 to 0.12 +### Unicode version strings -Nothing special. +While Versioneer works (and is continually tested) with both Python 2 and +Python 3, it is not entirely consistent with bytes-vs-unicode distinctions. +Newer releases probably generate unicode version strings on py2. It's not +clear that this is wrong, but it may be surprising for applications when then +write these strings to a network connection or include them in bytes-oriented +APIs like cryptographic checksums. -### Upgrading from 0.10 to 0.11 +[Bug #71](https://github.com/warner/python-versioneer/issues/71) investigates +this question. -You must add a `versioneer.VCS = "git"` to your `setup.py` before re-running -`setup.py setup_versioneer`. This will enable the use of additional -version-control systems (SVN, etc) in the future. + +## Updating Versioneer + +To upgrade your project to a new release of Versioneer, do the following: + +* install the new Versioneer (`pip install -U versioneer` or equivalent) +* edit `setup.cfg`, if necessary, to include any new configuration settings + indicated by the release notes. See [UPGRADING](./UPGRADING.md) for details. +* re-run `versioneer install` in your source tree, to replace + `SRC/_version.py` +* commit any changed files ## Future Directions @@ -333,9 +268,11 @@ ## License -To make Versioneer easier to embed, all its code is hereby released into the -public domain. The `_version.py` that it creates is also in the public -domain. +To make Versioneer easier to embed, all its code is dedicated to the public +domain. The `_version.py` that it creates is also in the public domain. +Specifically, both are released under the Creative Commons "Public Domain +Dedication" license (CC0-1.0), as described in +https://creativecommons.org/publicdomain/zero/1.0/ . """ @@ -353,12 +290,15 @@ class VersioneerConfig: - pass + """Container for Versioneer configuration parameters.""" def get_root(): - # we require that all commands are run from the project root, i.e. the - # directory that contains setup.py, setup.cfg, and versioneer.py . + """Get the project root directory. + + We require that all commands are run from the project root, i.e. the + directory that contains setup.py, setup.cfg, and versioneer.py . + """ root = os.path.realpath(os.path.abspath(os.getcwd())) setup_py = os.path.join(root, "setup.py") versioneer_py = os.path.join(root, "versioneer.py") @@ -382,7 +322,9 @@ def get_root(): # os.path.dirname(__file__), as that will find whichever # versioneer.py was first imported, even in later projects. me = os.path.realpath(os.path.abspath(__file__)) - if os.path.splitext(me)[0] != os.path.splitext(versioneer_py)[0]: + me_dir = os.path.normcase(os.path.splitext(me)[0]) + vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) + if me_dir != vsr_dir: print("Warning: build in %s is using versioneer.py from %s" % (os.path.dirname(me), versioneer_py)) except NameError: @@ -391,6 +333,7 @@ def get_root(): def get_config_from_root(root): + """Read the project setup.cfg file to determine Versioneer config.""" # This might raise EnvironmentError (if setup.cfg is missing), or # configparser.NoSectionError (if it lacks a [versioneer] section), or # configparser.NoOptionError (if it lacks "VCS="). See the docstring at @@ -411,13 +354,16 @@ def get(parser, name): cfg.versionfile_source = get(parser, "versionfile_source") cfg.versionfile_build = get(parser, "versionfile_build") cfg.tag_prefix = get(parser, "tag_prefix") + if cfg.tag_prefix in ("''", '""'): + cfg.tag_prefix = "" cfg.parentdir_prefix = get(parser, "parentdir_prefix") cfg.verbose = get(parser, "verbose") return cfg class NotThisMethod(Exception): - pass + """Exception raised if a method is not valid for the current scenario.""" + # these dictionaries contain VCS-specific tools LONG_VERSION_PY = {} @@ -425,7 +371,9 @@ class NotThisMethod(Exception): def register_vcs_handler(vcs, method): # decorator + """Decorator to mark a method as the handler for a particular VCS.""" def decorate(f): + """Store f in HANDLERS[vcs][method].""" if vcs not in HANDLERS: HANDLERS[vcs] = {} HANDLERS[vcs][method] = f @@ -433,14 +381,17 @@ def decorate(f): return decorate -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, + env=None): + """Call the given command(s).""" assert isinstance(commands, list) p = None for c in commands: try: dispcmd = str([c] + args) # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, + p = subprocess.Popen([c] + args, cwd=cwd, env=env, + stdout=subprocess.PIPE, stderr=(subprocess.PIPE if hide_stderr else None)) break @@ -451,19 +402,22 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): if verbose: print("unable to run %s" % dispcmd) print(e) - return None + return None, None else: if verbose: print("unable to find command, tried %s" % (commands,)) - return None + return None, None stdout = p.communicate()[0].strip() if sys.version_info[0] >= 3: stdout = stdout.decode() if p.returncode != 0: if verbose: print("unable to run %s (error)" % dispcmd) - return None - return stdout + print("stdout was %s" % stdout) + return None, p.returncode + return stdout, p.returncode + + LONG_VERSION_PY['git'] = ''' # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag @@ -472,7 +426,9 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): # that just contains the computed version number. # This file is released into the public domain. Generated by -# versioneer-0.15 (https://github.com/warner/python-versioneer) +# versioneer-0.18 (https://github.com/warner/python-versioneer) + +"""Git implementation of _version.py.""" import errno import os @@ -482,21 +438,24 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): def get_keywords(): + """Get the keywords needed to look up the version information.""" # these strings will be replaced by git during git-archive. # setup.py/versioneer.py will grep for the variable names, so they must # each be defined on a line of their own. _version.py will just call # get_keywords(). git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" - keywords = {"refnames": git_refnames, "full": git_full} + git_date = "%(DOLLAR)sFormat:%%ci%(DOLLAR)s" + keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} return keywords class VersioneerConfig: - pass + """Container for Versioneer configuration parameters.""" def get_config(): + """Create, populate and return the VersioneerConfig() object.""" # these strings are filled in when 'setup.py versioneer' creates # _version.py cfg = VersioneerConfig() @@ -510,7 +469,7 @@ def get_config(): class NotThisMethod(Exception): - pass + """Exception raised if a method is not valid for the current scenario.""" LONG_VERSION_PY = {} @@ -518,7 +477,9 @@ class NotThisMethod(Exception): def register_vcs_handler(vcs, method): # decorator + """Decorator to mark a method as the handler for a particular VCS.""" def decorate(f): + """Store f in HANDLERS[vcs][method].""" if vcs not in HANDLERS: HANDLERS[vcs] = {} HANDLERS[vcs][method] = f @@ -526,14 +487,17 @@ def decorate(f): return decorate -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, + env=None): + """Call the given command(s).""" assert isinstance(commands, list) p = None for c in commands: try: dispcmd = str([c] + args) # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, + p = subprocess.Popen([c] + args, cwd=cwd, env=env, + stdout=subprocess.PIPE, stderr=(subprocess.PIPE if hide_stderr else None)) break @@ -544,37 +508,50 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): if verbose: print("unable to run %%s" %% dispcmd) print(e) - return None + return None, None else: if verbose: print("unable to find command, tried %%s" %% (commands,)) - return None + return None, None stdout = p.communicate()[0].strip() if sys.version_info[0] >= 3: stdout = stdout.decode() if p.returncode != 0: if verbose: print("unable to run %%s (error)" %% dispcmd) - return None - return stdout + print("stdout was %%s" %% stdout) + return None, p.returncode + return stdout, p.returncode def versions_from_parentdir(parentdir_prefix, root, verbose): - # Source tarballs conventionally unpack into a directory that includes - # both the project name and a version string. - dirname = os.path.basename(root) - if not dirname.startswith(parentdir_prefix): - if verbose: - print("guessing rootdir is '%%s', but '%%s' doesn't start with " - "prefix '%%s'" %% (root, dirname, parentdir_prefix)) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None} + """Try to determine the version from the parent directory name. + + Source tarballs conventionally unpack into a directory that includes both + the project name and a version string. We will also support searching up + two directory levels for an appropriately named parent directory + """ + rootdirs = [] + + for i in range(3): + dirname = os.path.basename(root) + if dirname.startswith(parentdir_prefix): + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None, "date": None} + else: + rootdirs.append(root) + root = os.path.dirname(root) # up a level + + if verbose: + print("Tried directories %%s but none started with prefix %%s" %% + (str(rootdirs), parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") @register_vcs_handler("git", "get_keywords") def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" # the code embedded in _version.py can just fetch the value of these # keywords. When used from setup.py, we don't want to import _version.py, # so we do it with a regexp instead. This function is not used from @@ -591,6 +568,10 @@ def git_get_keywords(versionfile_abs): mo = re.search(r'=\s*"(.*)"', line) if mo: keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) f.close() except EnvironmentError: pass @@ -599,8 +580,18 @@ def git_get_keywords(versionfile_abs): @register_vcs_handler("git", "keywords") def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" if not keywords: raise NotThisMethod("no keywords at all, weird") + date = keywords.get("date") + if date is not None: + # git-2.2.0 added "%%cI", which expands to an ISO-8601 -compliant + # datestamp. However we prefer "%%ci" (which expands to an "ISO-8601 + # -like" string, which we must then edit to make compliant), because + # it's been around since git-1.5.3, and it's too difficult to + # discover which version we're using, or to work around using an + # older one. + date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) refnames = keywords["refnames"].strip() if refnames.startswith("$Format"): if verbose: @@ -621,7 +612,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # "stabilization", as well as "HEAD" and "master". tags = set([r for r in refs if re.search(r'\d', r)]) if verbose: - print("discarding '%%s', no digits" %% ",".join(refs-tags)) + print("discarding '%%s', no digits" %% ",".join(refs - tags)) if verbose: print("likely tags: %%s" %% ",".join(sorted(tags))) for ref in sorted(tags): @@ -632,41 +623,46 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): print("picking %%s" %% r) return {"version": r, "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None - } + "dirty": False, "error": None, + "date": date} # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") return {"version": "0+unknown", "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags"} + "dirty": False, "error": "no suitable tags", "date": None} @register_vcs_handler("git", "pieces_from_vcs") def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - # this runs 'git' from the root of the source tree. This only gets called - # if the git-archive 'subst' keywords were *not* expanded, and - # _version.py hasn't already been rewritten with a short version string, - # meaning we're inside a checked out source tree. - - if not os.path.exists(os.path.join(root, ".git")): - if verbose: - print("no .git in %%s" %% root) - raise NotThisMethod("no .git directory") + """Get version from 'git describe' in the root of the source tree. + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ GITS = ["git"] if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] - # if there is a tag, this yields TAG-NUM-gHEX[-dirty] - # if there are no tags, this yields HEX[-dirty] (no NUM) - describe_out = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long"], - cwd=root) + + out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, + hide_stderr=True) + if rc != 0: + if verbose: + print("Directory %%s not under git control" %% root) + raise NotThisMethod("'git rev-parse --git-dir' returned error") + + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%%s*" %% tag_prefix], + cwd=root) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") describe_out = describe_out.strip() - full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) if full_out is None: raise NotThisMethod("'git rev-parse' failed") full_out = full_out.strip() @@ -717,27 +713,34 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): else: # HEX: no tags pieces["closest-tag"] = None - count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) + count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) pieces["distance"] = int(count_out) # total number of commits + # commit date: see ISO-8601 comment in git_versions_from_keywords() + date = run_command(GITS, ["show", "-s", "--format=%%ci", "HEAD"], + cwd=root)[0].strip() + pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + return pieces def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" if "+" in pieces.get("closest-tag", ""): return "." return "+" def render_pep440(pieces): - # now build up version string, with post-release "local version - # identifier". Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - # get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + """Build up version string, with post-release "local version identifier". - # exceptions: - # 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: @@ -755,11 +758,11 @@ def render_pep440(pieces): def render_pep440_pre(pieces): - # TAG[.post.devDISTANCE] . No -dirty - - # exceptions: - # 1: no tags. 0.post.devDISTANCE + """TAG[.post.devDISTANCE] -- No -dirty. + Exceptions: + 1: no tags. 0.post.devDISTANCE + """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"]: @@ -771,14 +774,15 @@ def render_pep440_pre(pieces): def render_pep440_post(pieces): - # TAG[.postDISTANCE[.dev0]+gHEX] . The ".dev0" means dirty. Note that - # .dev0 sorts backwards (a dirty tree will appear "older" than the - # corresponding clean one), but you shouldn't be releasing software with - # -dirty anyways. + """TAG[.postDISTANCE[.dev0]+gHEX] . - # exceptions: - # 1: no tags. 0.postDISTANCE[.dev0] + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: @@ -797,11 +801,13 @@ def render_pep440_post(pieces): def render_pep440_old(pieces): - # TAG[.postDISTANCE[.dev0]] . The ".dev0" means dirty. + """TAG[.postDISTANCE[.dev0]] . - # exceptions: - # 1: no tags. 0.postDISTANCE[.dev0] + The ".dev0" means dirty. + Eexceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: @@ -817,12 +823,13 @@ def render_pep440_old(pieces): def render_git_describe(pieces): - # TAG[-DISTANCE-gHEX][-dirty], like 'git describe --tags --dirty - # --always' + """TAG[-DISTANCE-gHEX][-dirty]. - # exceptions: - # 1: no tags. HEX[-dirty] (note: no 'g' prefix) + Like 'git describe --tags --dirty --always'. + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"]: @@ -836,12 +843,14 @@ def render_git_describe(pieces): def render_git_describe_long(pieces): - # TAG-DISTANCE-gHEX[-dirty], like 'git describe --tags --dirty - # --always -long'. The distance/hash is unconditional. + """TAG-DISTANCE-gHEX[-dirty]. - # exceptions: - # 1: no tags. HEX[-dirty] (note: no 'g' prefix) + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) @@ -854,11 +863,13 @@ def render_git_describe_long(pieces): def render(pieces, style): + """Render the given version pieces into the requested style.""" if pieces["error"]: return {"version": "unknown", "full-revisionid": pieces.get("long"), "dirty": None, - "error": pieces["error"]} + "error": pieces["error"], + "date": None} if not style or style == "default": style = "pep440" # the default @@ -879,10 +890,12 @@ def render(pieces, style): raise ValueError("unknown style '%%s'" %% style) return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None} + "dirty": pieces["dirty"], "error": None, + "date": pieces.get("date")} def get_versions(): + """Get version information or return default if unable to do so.""" # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have # __file__, we can work backwards from there to the root. Some # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which @@ -907,7 +920,8 @@ def get_versions(): except NameError: return {"version": "0+unknown", "full-revisionid": None, "dirty": None, - "error": "unable to find root of source tree"} + "error": "unable to find root of source tree", + "date": None} try: pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) @@ -923,12 +937,13 @@ def get_versions(): return {"version": "0+unknown", "full-revisionid": None, "dirty": None, - "error": "unable to compute version"} + "error": "unable to compute version", "date": None} ''' @register_vcs_handler("git", "get_keywords") def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" # the code embedded in _version.py can just fetch the value of these # keywords. When used from setup.py, we don't want to import _version.py, # so we do it with a regexp instead. This function is not used from @@ -945,6 +960,10 @@ def git_get_keywords(versionfile_abs): mo = re.search(r'=\s*"(.*)"', line) if mo: keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) f.close() except EnvironmentError: pass @@ -953,8 +972,18 @@ def git_get_keywords(versionfile_abs): @register_vcs_handler("git", "keywords") def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" if not keywords: raise NotThisMethod("no keywords at all, weird") + date = keywords.get("date") + if date is not None: + # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant + # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 + # -like" string, which we must then edit to make compliant), because + # it's been around since git-1.5.3, and it's too difficult to + # discover which version we're using, or to work around using an + # older one. + date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) refnames = keywords["refnames"].strip() if refnames.startswith("$Format"): if verbose: @@ -975,7 +1004,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # "stabilization", as well as "HEAD" and "master". tags = set([r for r in refs if re.search(r'\d', r)]) if verbose: - print("discarding '%s', no digits" % ",".join(refs-tags)) + print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: print("likely tags: %s" % ",".join(sorted(tags))) for ref in sorted(tags): @@ -986,41 +1015,46 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): print("picking %s" % r) return {"version": r, "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None - } + "dirty": False, "error": None, + "date": date} # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") return {"version": "0+unknown", "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags"} + "dirty": False, "error": "no suitable tags", "date": None} @register_vcs_handler("git", "pieces_from_vcs") def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - # this runs 'git' from the root of the source tree. This only gets called - # if the git-archive 'subst' keywords were *not* expanded, and - # _version.py hasn't already been rewritten with a short version string, - # meaning we're inside a checked out source tree. - - if not os.path.exists(os.path.join(root, ".git")): - if verbose: - print("no .git in %s" % root) - raise NotThisMethod("no .git directory") + """Get version from 'git describe' in the root of the source tree. + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ GITS = ["git"] if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] - # if there is a tag, this yields TAG-NUM-gHEX[-dirty] - # if there are no tags, this yields HEX[-dirty] (no NUM) - describe_out = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long"], - cwd=root) + + out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, + hide_stderr=True) + if rc != 0: + if verbose: + print("Directory %s not under git control" % root) + raise NotThisMethod("'git rev-parse --git-dir' returned error") + + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%s*" % tag_prefix], + cwd=root) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") describe_out = describe_out.strip() - full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) if full_out is None: raise NotThisMethod("'git rev-parse' failed") full_out = full_out.strip() @@ -1071,14 +1105,24 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): else: # HEX: no tags pieces["closest-tag"] = None - count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) + count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) pieces["distance"] = int(count_out) # total number of commits + # commit date: see ISO-8601 comment in git_versions_from_keywords() + date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], + cwd=root)[0].strip() + pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + return pieces def do_vcs_install(manifest_in, versionfile_source, ipy): + """Git-specific installation logic for Versioneer. + + For Git, this means creating/changing .gitattributes to mark _version.py + for export-subst keyword substitution. + """ GITS = ["git"] if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] @@ -1112,26 +1156,37 @@ def do_vcs_install(manifest_in, versionfile_source, ipy): def versions_from_parentdir(parentdir_prefix, root, verbose): - # Source tarballs conventionally unpack into a directory that includes - # both the project name and a version string. - dirname = os.path.basename(root) - if not dirname.startswith(parentdir_prefix): - if verbose: - print("guessing rootdir is '%s', but '%s' doesn't start with " - "prefix '%s'" % (root, dirname, parentdir_prefix)) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None} + """Try to determine the version from the parent directory name. + + Source tarballs conventionally unpack into a directory that includes both + the project name and a version string. We will also support searching up + two directory levels for an appropriately named parent directory + """ + rootdirs = [] + + for i in range(3): + dirname = os.path.basename(root) + if dirname.startswith(parentdir_prefix): + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None, "date": None} + else: + rootdirs.append(root) + root = os.path.dirname(root) # up a level + + if verbose: + print("Tried directories %s but none started with prefix %s" % + (str(rootdirs), parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + SHORT_VERSION_PY = """ -# This file was generated by 'versioneer.py' (0.15) from +# This file was generated by 'versioneer.py' (0.18) from # revision-control system data, or from the parent directory name of an # unpacked source archive. Distribution tarballs contain a pre-generated copy # of this file. import json -import sys version_json = ''' %s @@ -1144,6 +1199,7 @@ def get_versions(): def versions_from_file(filename): + """Try to determine the version from _version.py if present.""" try: with open(filename) as f: contents = f.read() @@ -1151,12 +1207,16 @@ def versions_from_file(filename): raise NotThisMethod("unable to read _version.py") mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", contents, re.M | re.S) + if not mo: + mo = re.search(r"version_json = '''\r\n(.*)''' # END VERSION_JSON", + contents, re.M | re.S) if not mo: raise NotThisMethod("no version_json in _version.py") return json.loads(mo.group(1)) def write_to_version_file(filename, versions): + """Write the given version number to the given _version.py file.""" os.unlink(filename) contents = json.dumps(versions, sort_keys=True, indent=1, separators=(",", ": ")) @@ -1167,19 +1227,21 @@ def write_to_version_file(filename, versions): def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" if "+" in pieces.get("closest-tag", ""): return "." return "+" def render_pep440(pieces): - # now build up version string, with post-release "local version - # identifier". Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - # get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + """Build up version string, with post-release "local version identifier". - # exceptions: - # 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: @@ -1197,11 +1259,11 @@ def render_pep440(pieces): def render_pep440_pre(pieces): - # TAG[.post.devDISTANCE] . No -dirty - - # exceptions: - # 1: no tags. 0.post.devDISTANCE + """TAG[.post.devDISTANCE] -- No -dirty. + Exceptions: + 1: no tags. 0.post.devDISTANCE + """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"]: @@ -1213,14 +1275,15 @@ def render_pep440_pre(pieces): def render_pep440_post(pieces): - # TAG[.postDISTANCE[.dev0]+gHEX] . The ".dev0" means dirty. Note that - # .dev0 sorts backwards (a dirty tree will appear "older" than the - # corresponding clean one), but you shouldn't be releasing software with - # -dirty anyways. + """TAG[.postDISTANCE[.dev0]+gHEX] . - # exceptions: - # 1: no tags. 0.postDISTANCE[.dev0] + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: @@ -1239,11 +1302,13 @@ def render_pep440_post(pieces): def render_pep440_old(pieces): - # TAG[.postDISTANCE[.dev0]] . The ".dev0" means dirty. + """TAG[.postDISTANCE[.dev0]] . - # exceptions: - # 1: no tags. 0.postDISTANCE[.dev0] + The ".dev0" means dirty. + Eexceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: @@ -1259,12 +1324,13 @@ def render_pep440_old(pieces): def render_git_describe(pieces): - # TAG[-DISTANCE-gHEX][-dirty], like 'git describe --tags --dirty - # --always' + """TAG[-DISTANCE-gHEX][-dirty]. - # exceptions: - # 1: no tags. HEX[-dirty] (note: no 'g' prefix) + Like 'git describe --tags --dirty --always'. + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"]: @@ -1278,12 +1344,14 @@ def render_git_describe(pieces): def render_git_describe_long(pieces): - # TAG-DISTANCE-gHEX[-dirty], like 'git describe --tags --dirty - # --always -long'. The distance/hash is unconditional. + """TAG-DISTANCE-gHEX[-dirty]. - # exceptions: - # 1: no tags. HEX[-dirty] (note: no 'g' prefix) + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) @@ -1296,11 +1364,13 @@ def render_git_describe_long(pieces): def render(pieces, style): + """Render the given version pieces into the requested style.""" if pieces["error"]: return {"version": "unknown", "full-revisionid": pieces.get("long"), "dirty": None, - "error": pieces["error"]} + "error": pieces["error"], + "date": None} if not style or style == "default": style = "pep440" # the default @@ -1321,16 +1391,19 @@ def render(pieces, style): raise ValueError("unknown style '%s'" % style) return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None} + "dirty": pieces["dirty"], "error": None, + "date": pieces.get("date")} class VersioneerBadRootError(Exception): - pass + """The project root directory is unknown or missing key files.""" def get_versions(verbose=False): - # returns dict with two keys: 'version' and 'full' + """Get the project version from whatever source is available. + Returns dict with two keys: 'version' and 'full'. + """ if "versioneer" in sys.modules: # see the discussion in cmdclass.py:get_cmdclass() del sys.modules["versioneer"] @@ -1398,14 +1471,17 @@ def get_versions(verbose=False): print("unable to compute version") return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, "error": "unable to compute version"} + "dirty": None, "error": "unable to compute version", + "date": None} def get_version(): + """Get the short version string for this project.""" return get_versions()["version"] def get_cmdclass(): + """Get the custom setuptools/distutils subclasses used by Versioneer.""" if "versioneer" in sys.modules: del sys.modules["versioneer"] # this fixes the "python setup.py develop" case (also 'install' and @@ -1442,6 +1518,7 @@ def run(self): print("Version: %s" % vers["version"]) print(" full-revisionid: %s" % vers.get("full-revisionid")) print(" dirty: %s" % vers.get("dirty")) + print(" date: %s" % vers.get("date")) if vers["error"]: print(" error: %s" % vers["error"]) cmds["version"] = cmd_version @@ -1455,8 +1532,17 @@ def run(self): # setuptools/bdist_egg -> distutils/install_lib -> build_py # setuptools/install -> bdist_egg ->.. # setuptools/develop -> ? + # pip install: + # copies source tree to a tempdir before running egg_info/etc + # if .git isn't copied too, 'git describe' will fail + # then does setup.py bdist_wheel, or sometimes setup.py install + # setup.py egg_info -> ? - from distutils.command.build_py import build_py as _build_py + # we override different "build_py" commands for both environments + if "setuptools" in sys.modules: + from setuptools.command.build_py import build_py as _build_py + else: + from distutils.command.build_py import build_py as _build_py class cmd_build_py(_build_py): def run(self): @@ -1475,6 +1561,12 @@ def run(self): if "cx_Freeze" in sys.modules: # cx_freeze enabled? from cx_Freeze.dist import build_exe as _build_exe + # nczeczulin reports that py2exe won't like the pep440-style string + # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. + # setup(console=[{ + # "version": versioneer.get_version().split("+", 1)[0], # FILEVERSION + # "product_version": versioneer.get_version(), + # ... class cmd_build_exe(_build_exe): def run(self): @@ -1499,6 +1591,34 @@ def run(self): cmds["build_exe"] = cmd_build_exe del cmds["build_py"] + if 'py2exe' in sys.modules: # py2exe enabled? + try: + from py2exe.distutils_buildexe import py2exe as _py2exe # py3 + except ImportError: + from py2exe.build_exe import py2exe as _py2exe # py2 + + class cmd_py2exe(_py2exe): + def run(self): + root = get_root() + cfg = get_config_from_root(root) + versions = get_versions() + target_versionfile = cfg.versionfile_source + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, versions) + + _py2exe.run(self) + os.unlink(target_versionfile) + with open(cfg.versionfile_source, "w") as f: + LONG = LONG_VERSION_PY[cfg.VCS] + f.write(LONG % + {"DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + }) + cmds["py2exe"] = cmd_py2exe + # we override different "sdist" commands for both environments if "setuptools" in sys.modules: from setuptools.command.sdist import sdist as _sdist @@ -1539,7 +1659,7 @@ def make_release_tree(self, base_dir, files): style = pep440 versionfile_source = src/myproject/_version.py versionfile_build = myproject/_version.py - tag_prefix = "" + tag_prefix = parentdir_prefix = myproject- You will also need to edit your setup.py to use the results: @@ -1575,6 +1695,7 @@ def make_release_tree(self, base_dir, files): def do_setup(): + """Main VCS-independent setup function for installing Versioneer.""" root = get_root() try: cfg = get_config_from_root(root) @@ -1649,13 +1770,14 @@ def do_setup(): print(" versionfile_source already in MANIFEST.in") # Make VCS-specific changes. For git, this means creating/changing - # .gitattributes to mark _version.py for export-time keyword + # .gitattributes to mark _version.py for export-subst keyword # substitution. do_vcs_install(manifest_in, cfg.versionfile_source, ipy) return 0 def scan_setup_py(): + """Validate the contents of setup.py against Versioneer's expectations.""" found = set() setters = False errors = 0 @@ -1690,6 +1812,7 @@ def scan_setup_py(): errors += 1 return errors + if __name__ == "__main__": cmd = sys.argv[1] if cmd == "setup": From 7b14f3e8222c86c651c0654a466c98e97ec9d313 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 22 Feb 2017 16:12:20 -0500 Subject: [PATCH 025/519] don't use a prefix in setup.cfg --- packages/pandas-gbq/pandas_gbq/__init__.py | 6 +----- packages/pandas-gbq/pandas_gbq/_version.py | 4 ++-- packages/pandas-gbq/setup.cfg | 2 +- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/__init__.py b/packages/pandas-gbq/pandas_gbq/__init__.py index fc9cab62fd60..c689b0d1d24e 100644 --- a/packages/pandas-gbq/pandas_gbq/__init__.py +++ b/packages/pandas-gbq/pandas_gbq/__init__.py @@ -2,10 +2,6 @@ from ._version import get_versions versions = get_versions() -__version__ = versions['version'] +__version__ = versions.get('closest-tag', versions['version']) __git_revision__ = versions['full-revisionid'] del get_versions, versions - -from ._version import get_versions -__version__ = get_versions()['version'] -del get_versions diff --git a/packages/pandas-gbq/pandas_gbq/_version.py b/packages/pandas-gbq/pandas_gbq/_version.py index 6f81e4ccfade..e4e698ea4973 100644 --- a/packages/pandas-gbq/pandas_gbq/_version.py +++ b/packages/pandas-gbq/pandas_gbq/_version.py @@ -41,8 +41,8 @@ def get_config(): cfg = VersioneerConfig() cfg.VCS = "git" cfg.style = "pep440" - cfg.tag_prefix = "v" - cfg.parentdir_prefix = "pandas_gbq-" + cfg.tag_prefix = "" + cfg.parentdir_prefix = "pandas_gbq/_version.py" cfg.versionfile_source = "pandas_gbq/_version.py" cfg.verbose = False return cfg diff --git a/packages/pandas-gbq/setup.cfg b/packages/pandas-gbq/setup.cfg index cab8f5b77750..18c520a8669f 100644 --- a/packages/pandas-gbq/setup.cfg +++ b/packages/pandas-gbq/setup.cfg @@ -8,7 +8,7 @@ VCS = git style = pep440 versionfile_source = pandas_gbq/_version.py versionfile_build = pandas_gbq/_version.py -tag_prefix = v +tag_prefix = parentdir_prefix = pandas_gbq- [flake8] From 848eb322227430c8b4bc4f4f847b844e243892e9 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Thu, 23 Feb 2017 07:47:18 -0500 Subject: [PATCH 026/519] fix up setup.py release 0.1.2 --- packages/pandas-gbq/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index c7dd71f2fcea..3aa08a64b29f 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -24,6 +24,7 @@ def readme(): setup( name=NAME, version=versioneer.get_version(), + cmdclass=versioneer.get_cmdclass(), description="Pandas interface to Google Big Query", long_description=readme(), license='BSD License', @@ -48,5 +49,4 @@ def readme(): install_requires=INSTALL_REQUIRES, packages=find_packages(exclude=['contrib', 'docs', 'tests*']), test_suite='tests', - zip_safe=False, ) From be0a96b45c91df006cf53ca3918dbde9a2f2b4a0 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Thu, 23 Feb 2017 09:18:49 -0500 Subject: [PATCH 027/519] add requirements.txt file --- packages/pandas-gbq/requirements.txt | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 packages/pandas-gbq/requirements.txt diff --git a/packages/pandas-gbq/requirements.txt b/packages/pandas-gbq/requirements.txt new file mode 100644 index 000000000000..11bb601852d4 --- /dev/null +++ b/packages/pandas-gbq/requirements.txt @@ -0,0 +1,4 @@ +pandas +httplib2 +google-api-python-client +oauth2client From 060dd3ebf16b874e5c789c5829a6fc7e6d30e139 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Thu, 23 Feb 2017 09:22:22 -0500 Subject: [PATCH 028/519] update install docs --- packages/pandas-gbq/docs/source/install.rst | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/packages/pandas-gbq/docs/source/install.rst b/packages/pandas-gbq/docs/source/install.rst index b7dfd560831a..a059f725a2e1 100644 --- a/packages/pandas-gbq/docs/source/install.rst +++ b/packages/pandas-gbq/docs/source/install.rst @@ -1,26 +1,24 @@ Install pandas-gbq ================== -You can install pandas-gbq with ``conda``, with ``pip``, or by installing from source. +You can install pandas-gbq with ``pip``, or by installing from source. -Conda ------ - -To install the latest version of Dask from the -`conda-forge `_ repository using -`conda `_:: +Pip +--- - conda install pandas-gbq -c conda-forge +To install the latest version of pandas-gbq: from the -This installs dask and all common dependencies, including Pandas and NumPy. + pip install pandas-gbq -Pip ---- +This installs pandas-gbq and all common dependencies, including Pandas. Install from Source ------------------- +.. code-block:: shell + + $ pip install git+https://github.com/pydata/pandas-gbq.git Test From 83a0a26fa8e10f9ef072cf74e58f4f765cf70e46 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Thu, 23 Feb 2017 09:24:50 -0500 Subject: [PATCH 029/519] DOC: doc typo --- packages/pandas-gbq/docs/source/install.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/docs/source/install.rst b/packages/pandas-gbq/docs/source/install.rst index a059f725a2e1..ea81060e61bb 100644 --- a/packages/pandas-gbq/docs/source/install.rst +++ b/packages/pandas-gbq/docs/source/install.rst @@ -8,7 +8,9 @@ Pip To install the latest version of pandas-gbq: from the - pip install pandas-gbq +.. code-block:: shell + + $ pip install pandas-gbq -U This installs pandas-gbq and all common dependencies, including Pandas. From 2a8c1df4baaaf5266ae1be5e17968ddb8235ec17 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Sun, 26 Feb 2017 13:46:01 -0500 Subject: [PATCH 030/519] CI: add in codecov.yml file --- packages/pandas-gbq/codecov.yml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 packages/pandas-gbq/codecov.yml diff --git a/packages/pandas-gbq/codecov.yml b/packages/pandas-gbq/codecov.yml new file mode 100644 index 000000000000..59f8fdf6156f --- /dev/null +++ b/packages/pandas-gbq/codecov.yml @@ -0,0 +1,9 @@ +coverage: + status: + project: + default: + target: '35' + patch: + default: + target: '50' + branches: null From 91a731cb1d5ab8dcf90b2c94bdd8fb67a4c7898c Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Sun, 26 Feb 2017 13:52:49 -0500 Subject: [PATCH 031/519] add changelog to docs --- packages/pandas-gbq/docs/source/_static/style.css | 3 +++ .../pandas-gbq/docs/source/_templates/layout.html | 2 ++ packages/pandas-gbq/docs/source/changelog.rst | 15 +++++++++++++++ packages/pandas-gbq/docs/source/conf.py | 15 ++++++++++++--- packages/pandas-gbq/docs/source/index.rst | 1 + 5 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 packages/pandas-gbq/docs/source/_static/style.css create mode 100644 packages/pandas-gbq/docs/source/_templates/layout.html create mode 100644 packages/pandas-gbq/docs/source/changelog.rst diff --git a/packages/pandas-gbq/docs/source/_static/style.css b/packages/pandas-gbq/docs/source/_static/style.css new file mode 100644 index 000000000000..7f69caf30b48 --- /dev/null +++ b/packages/pandas-gbq/docs/source/_static/style.css @@ -0,0 +1,3 @@ +@import url("theme.css"); + +a.internal em {font-style: normal} diff --git a/packages/pandas-gbq/docs/source/_templates/layout.html b/packages/pandas-gbq/docs/source/_templates/layout.html new file mode 100644 index 000000000000..4c57ba83056d --- /dev/null +++ b/packages/pandas-gbq/docs/source/_templates/layout.html @@ -0,0 +1,2 @@ +{% extends "!layout.html" %} +{% set css_files = css_files + ["_static/style.css"] %} diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst new file mode 100644 index 000000000000..3ec2a3df748b --- /dev/null +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -0,0 +1,15 @@ +Changelog +========= + +0.2.0 / 2017-? +-------------- + +0.1.2 / 2017-02-23 +------------------ + +Initial release of transfered code from `pandas `__ + +Includes patches since the 0.19.2 release on pandas with the following: + +- ``read_gbq`` now allows query configuration preferences `here `__ +- ``read_gbq`` now stores ``INTEGER`` columns as ``dtype=object`` if they contain ``NULL`` values. Otherwise they are stored as ``int64``. This prevents precision lost for integers greather than 2**53. Furthermore ``FLOAT`` columns with values above 10**4 are no longer casted to ``int64`` which also caused precision loss `here `__, and `here `__ diff --git a/packages/pandas-gbq/docs/source/conf.py b/packages/pandas-gbq/docs/source/conf.py index 81367a6201f1..32c2fcfcb5c3 100644 --- a/packages/pandas-gbq/docs/source/conf.py +++ b/packages/pandas-gbq/docs/source/conf.py @@ -16,8 +16,8 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # -# import os -# import sys +import os +import sys # sys.path.insert(0, os.path.abspath('.')) # -- General configuration ------------------------------------------------ @@ -118,10 +118,19 @@ # -- Options for HTML output ---------------------------------------------- +# Taken from docs.readthedocs.io: +# on_rtd is whether we are on readthedocs.io +on_rtd = os.environ.get('READTHEDOCS', None) == 'True' + +if not on_rtd: # only import and set the theme if we're building docs locally + import sphinx_rtd_theme + html_theme = 'sphinx_rtd_theme' + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] + # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' +# html_theme = 'alabaster' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/packages/pandas-gbq/docs/source/index.rst b/packages/pandas-gbq/docs/source/index.rst index cb6d4f56ae15..01e07c480725 100644 --- a/packages/pandas-gbq/docs/source/index.rst +++ b/packages/pandas-gbq/docs/source/index.rst @@ -13,6 +13,7 @@ Contents: install.rst examples-tutorials.rst + changelog.rst Indices and tables From 4d4cf09ab759ec844f682cdbaf91de84ea3dd137 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Sun, 26 Feb 2017 15:16:45 -0500 Subject: [PATCH 032/519] add merging pr script --- packages/pandas-gbq/scripts/merge-py.py | 264 ++++++++++++++++++++++++ 1 file changed, 264 insertions(+) create mode 100755 packages/pandas-gbq/scripts/merge-py.py diff --git a/packages/pandas-gbq/scripts/merge-py.py b/packages/pandas-gbq/scripts/merge-py.py new file mode 100755 index 000000000000..3596308e0a37 --- /dev/null +++ b/packages/pandas-gbq/scripts/merge-py.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Utility for creating well-formed pull request merges and pushing them to +# Apache. +# usage: ./apache-pr-merge.py (see config env vars below) +# +# Lightly modified from version of this script in incubator-parquet-format + +from __future__ import print_function + +from subprocess import check_output +from requests.auth import HTTPBasicAuth +import requests + +import os +import six +import sys +import textwrap + +from six.moves import input + +PANDASGBQ_HOME = '.' +PROJECT_NAME = 'pandas-gbq' +print("PANDASGBQ_HOME = " + PANDASGBQ_HOME) + +# Remote name with the PR +PR_REMOTE_NAME = os.environ.get("PR_REMOTE_NAME", "upstream") + +# Remote name where results pushed +PUSH_REMOTE_NAME = os.environ.get("PUSH_REMOTE_NAME", "upstream") + +GITHUB_BASE = "https://github.com/pydata/" + PROJECT_NAME + "/pull" +GITHUB_API_BASE = "https://api.github.com/repos/pydata/" + PROJECT_NAME + +# Prefix added to temporary branches +BRANCH_PREFIX = "PR_TOOL" + +os.chdir(PANDASGBQ_HOME) + +auth_required = False + +if auth_required: + GITHUB_USERNAME = os.environ['GITHUB_USER'] + import getpass + GITHUB_PASSWORD = getpass.getpass('Enter github.com password for %s:' + % GITHUB_USERNAME) + + def get_json_auth(url): + auth = HTTPBasicAuth(GITHUB_USERNAME, GITHUB_PASSWORD) + req = requests.get(url, auth=auth) + return req.json() + + get_json = get_json_auth +else: + def get_json_no_auth(url): + req = requests.get(url) + return req.json() + + get_json = get_json_no_auth + + +def fail(msg): + print(msg) + clean_up() + sys.exit(-1) + + +def run_cmd(cmd): + if isinstance(cmd, six.string_types): + cmd = cmd.split(' ') + + output = check_output(cmd) + + if isinstance(output, six.binary_type): + output = output.decode('utf-8') + return output + + +def continue_maybe(prompt): + result = input("\n%s (y/n): " % prompt) + if result.lower() != "y": + fail("Okay, exiting") + + +original_head = run_cmd("git rev-parse HEAD")[:8] + + +def clean_up(): + print("Restoring head pointer to %s" % original_head) + run_cmd("git checkout %s" % original_head) + + branches = run_cmd("git branch").replace(" ", "").split("\n") + + for branch in [b for b in branches if b.startswith(BRANCH_PREFIX)]: + print("Deleting local branch %s" % branch) + run_cmd("git branch -D %s" % branch) + + +# Merge the requested PR and return the merge hash +def merge_pr(pr_num, target_ref): + + pr_branch_name = "%s_MERGE_PR_%s" % (BRANCH_PREFIX, pr_num) + target_branch_name = "%s_MERGE_PR_%s_%s" % (BRANCH_PREFIX, pr_num, + target_ref.upper()) + run_cmd("git fetch %s pull/%s/head:%s" % (PR_REMOTE_NAME, pr_num, + pr_branch_name)) + run_cmd("git fetch %s %s:%s" % (PUSH_REMOTE_NAME, target_ref, + target_branch_name)) + run_cmd("git checkout %s" % target_branch_name) + + had_conflicts = False + try: + run_cmd(['git', 'merge', pr_branch_name, '--squash']) + except Exception as e: + msg = ("Error merging: %s\nWould you like to manually fix-up " + "this merge?" % e) + continue_maybe(msg) + msg = ("Okay, please fix any conflicts and 'git add' " + "conflicting files... Finished?") + continue_maybe(msg) + had_conflicts = True + + commit_authors = run_cmd(['git', 'log', 'HEAD..%s' % pr_branch_name, + '--pretty=format:%an <%ae>']).split("\n") + distinct_authors = sorted(set(commit_authors), + key=lambda x: commit_authors.count(x), + reverse=True) + primary_author = distinct_authors[0] + commits = run_cmd(['git', 'log', 'HEAD..%s' % pr_branch_name, + '--pretty=format:%h [%an] %s']).split("\n\n") + + merge_message_flags = [] + + merge_message_flags += ["-m", title] + if body is not None: + merge_message_flags += ["-m", '\n'.join(textwrap.wrap(body))] + + authors = "\n".join(["Author: %s" % a for a in distinct_authors]) + + merge_message_flags += ["-m", authors] + + if had_conflicts: + committer_name = run_cmd("git config --get user.name").strip() + committer_email = run_cmd("git config --get user.email").strip() + message = ("This patch had conflicts when merged, " + "resolved by\nCommitter: %s <%s>" + % (committer_name, committer_email)) + merge_message_flags += ["-m", message] + + # The string "Closes #%s" string is required for GitHub to correctly close + # the PR + merge_message_flags += [ + "-m", + "Closes #%s from %s and squashes the following commits:" + % (pr_num, pr_repo_desc)] + for c in commits: + merge_message_flags += ["-m", c] + + run_cmd(['git', 'commit', '--author="%s"' % primary_author] + + merge_message_flags) + + continue_maybe("Merge complete (local ref %s). Push to %s?" % ( + target_branch_name, PUSH_REMOTE_NAME)) + + try: + run_cmd('git push %s %s:%s' % (PUSH_REMOTE_NAME, target_branch_name, + target_ref)) + except Exception as e: + clean_up() + fail("Exception while pushing: %s" % e) + + merge_hash = run_cmd("git rev-parse %s" % target_branch_name)[:8] + clean_up() + print("Pull request #%s merged!" % pr_num) + print("Merge hash: %s" % merge_hash) + return merge_hash + + +def cherry_pick(pr_num, merge_hash, default_branch): + pick_ref = input("Enter a branch name [%s]: " % default_branch) + if pick_ref == "": + pick_ref = default_branch + + pick_branch_name = "%s_PICK_PR_%s_%s" % (BRANCH_PREFIX, pr_num, + pick_ref.upper()) + + run_cmd("git fetch %s %s:%s" % (PUSH_REMOTE_NAME, pick_ref, + pick_branch_name)) + run_cmd("git checkout %s" % pick_branch_name) + run_cmd("git cherry-pick -sx %s" % merge_hash) + + continue_maybe("Pick complete (local ref %s). Push to %s?" % ( + pick_branch_name, PUSH_REMOTE_NAME)) + + try: + run_cmd('git push %s %s:%s' % (PUSH_REMOTE_NAME, pick_branch_name, + pick_ref)) + except Exception as e: + clean_up() + fail("Exception while pushing: %s" % e) + + pick_hash = run_cmd("git rev-parse %s" % pick_branch_name)[:8] + clean_up() + + print("Pull request #%s picked into %s!" % (pr_num, pick_ref)) + print("Pick hash: %s" % pick_hash) + return pick_ref + + +def fix_version_from_branch(branch, versions): + # Note: Assumes this is a sorted (newest->oldest) list of un-released + # versions + if branch == "master": + return versions[0] + else: + branch_ver = branch.replace("branch-", "") + return filter(lambda x: x.name.startswith(branch_ver), versions)[-1] + +pr_num = input("Which pull request would you like to merge? (e.g. 34): ") +pr = get_json("%s/pulls/%s" % (GITHUB_API_BASE, pr_num)) + +url = pr["url"] +title = pr["title"] +body = pr["body"] +target_ref = pr["base"]["ref"] +user_login = pr["user"]["login"] +base_ref = pr["head"]["ref"] +pr_repo_desc = "%s/%s" % (user_login, base_ref) + +if pr["merged"] is True: + print("Pull request {0} has already been merged, please backport manually" + .format(pr_num)) + sys.exit(0) + +if not bool(pr["mergeable"]): + msg = ("Pull request {0} is not mergeable in its current form.\n" + "Continue? (experts only!)".format(pr_num)) + continue_maybe(msg) + +print("\n=== Pull Request #%s ===" % pr_num) +print("title\t%s\nsource\t%s\ntarget\t%s\nurl\t%s" + % (title, pr_repo_desc, target_ref, url)) +continue_maybe("Proceed with merging pull request #%s?" % pr_num) + +merged_refs = [target_ref] + +merge_hash = merge_pr(pr_num, target_ref) From 6cb29073e04d5df7422e9680ffb9af06a84a246c Mon Sep 17 00:00:00 2001 From: Matti Remes Date: Sun, 26 Feb 2017 15:24:10 -0500 Subject: [PATCH 033/519] Use name and type comparising when appending a dataframe into table I modified GbqConnector.verify_schema function to parse name and type from the remote schema (basically dropping mode) and include those in the compared fields. Currently, when appending to a BQ table, comparison between the destination table's schema and a dataframe schema is done over superset of a BQ schema definition (name, type, mode) when _generate_bq_schema parses only name and type from a dataframe. IMO it would be inconvenient to make the mode check in the module by generating completeness of columns (includes null values or not). So raising a generic GBQ error is more convenient here. closes #13 Author: Matti Remes Closes #14 from mremes/master and squashes the following commits: bf8c378 [Matti Remes] added reference to issue #13 77b1fd5 [Matti Remes] changelog for verify_schema changes 70d08ef [Matti Remes] make the syntax of the test flake-pretty 45826f1 [Matti Remes] Merge remote-tracking branch 'upstream/master' 66aa616 [Matti Remes] Added test for validate_schema ignoring field mode when comparing schemas 5dafd55 [Matti Remes] fix bug with selecting key 631d66c [Matti Remes] Use name and type of fields for comparing remote and local schemas when appending to a table --- packages/pandas-gbq/docs/source/changelog.rst | 6 ++-- packages/pandas-gbq/docs/source/conf.py | 5 ++-- packages/pandas-gbq/pandas_gbq/gbq.py | 6 +++- .../pandas-gbq/pandas_gbq/tests/test_gbq.py | 28 +++++++++++++++++++ 4 files changed, 39 insertions(+), 6 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 3ec2a3df748b..b2a6b83e46d1 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,8 +1,10 @@ Changelog ========= -0.2.0 / 2017-? --------------- +0.2.0 / 2017-03-?? +------------------ + +- Bug with appending to a BigQuery table where fields have modes (NULLABLE,REQUIRED,REPEATED) specified. These modes were compared versus the remote schema and writing a table via ``to_gbq`` would previously raise. (:issue:`13`) 0.1.2 / 2017-02-23 ------------------ diff --git a/packages/pandas-gbq/docs/source/conf.py b/packages/pandas-gbq/docs/source/conf.py index 32c2fcfcb5c3..94c8d2296055 100644 --- a/packages/pandas-gbq/docs/source/conf.py +++ b/packages/pandas-gbq/docs/source/conf.py @@ -353,6 +353,5 @@ intersphinx_mapping = {'https://docs.python.org/': None} extlinks = {'issue': ('https://github.com/pydata/pandas-gbq/issues/%s', - 'GH'), - 'wiki': ('https://github.com/pydata/pandas-gbq/wiki/%s', - 'wiki ')} + 'GH#'), + 'pr': ('https://github.com/pydata/pandas-gbq/pull/%s', 'GH#')} diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 9759e37932b8..060724ed8de2 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -563,8 +563,12 @@ def verify_schema(self, dataset_id, table_id, schema): datasetId=dataset_id, tableId=table_id).execute()['schema'] + remote_fields = [{'name': field_remote['name'], + 'type': field_remote['type']} + for field_remote in remote_schema['fields']] + fields_remote = set([json.dumps(field_remote) - for field_remote in remote_schema['fields']]) + for field_remote in remote_fields]) fields_local = set(json.dumps(field_local) for field_local in schema['fields']) diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py index 036e833051f1..6a3cad19f83b 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -1161,6 +1161,34 @@ def test_upload_data_flexible_column_order(self): _get_project_id(), if_exists='append', private_key=_get_private_key_path()) + def test_verify_schema_ignores_field_mode(self): + test_id = "14" + test_schema_1 = {'fields': [{'name': 'A', + 'type': 'FLOAT', + 'mode': 'NULLABLE'}, + {'name': 'B', + 'type': 'FLOAT', + 'mode': 'NULLABLE'}, + {'name': 'C', + 'type': 'STRING', + 'mode': 'NULLABLE'}, + {'name': 'D', + 'type': 'TIMESTAMP', + 'mode': 'REQUIRED'}]} + test_schema_2 = {'fields': [{'name': 'A', + 'type': 'FLOAT'}, + {'name': 'B', + 'type': 'FLOAT'}, + {'name': 'C', + 'type': 'STRING'}, + {'name': 'D', + 'type': 'TIMESTAMP'}]} + + self.table.create(TABLE_ID + test_id, test_schema_1) + self.assertTrue(self.sut.verify_schema( + self.dataset_prefix + "1", TABLE_ID + test_id, test_schema_2), + 'Expected schema to match') + def test_list_dataset(self): dataset_id = self.dataset_prefix + "1" self.assertTrue(dataset_id in self.dataset.datasets(), From 093d950c9afc06e8cf226630168cc3ad88a3d983 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Sun, 26 Feb 2017 15:43:28 -0500 Subject: [PATCH 034/519] move pandas docs --- packages/pandas-gbq/docs/source/conf.py | 15 ++- .../docs/source/examples-tutorials.rst | 2 - packages/pandas-gbq/docs/source/index.rst | 18 +++- packages/pandas-gbq/docs/source/install.rst | 10 +- packages/pandas-gbq/docs/source/intro.rst | 64 +++++++++++++ packages/pandas-gbq/docs/source/reading.rst | 57 ++++++++++++ packages/pandas-gbq/docs/source/tables.rst | 22 +++++ packages/pandas-gbq/docs/source/writing.rst | 91 +++++++++++++++++++ 8 files changed, 271 insertions(+), 8 deletions(-) delete mode 100644 packages/pandas-gbq/docs/source/examples-tutorials.rst create mode 100644 packages/pandas-gbq/docs/source/intro.rst create mode 100644 packages/pandas-gbq/docs/source/reading.rst create mode 100644 packages/pandas-gbq/docs/source/tables.rst create mode 100644 packages/pandas-gbq/docs/source/writing.rst diff --git a/packages/pandas-gbq/docs/source/conf.py b/packages/pandas-gbq/docs/source/conf.py index 94c8d2296055..71cff5254d98 100644 --- a/packages/pandas-gbq/docs/source/conf.py +++ b/packages/pandas-gbq/docs/source/conf.py @@ -29,9 +29,18 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = [ - 'sphinx.ext.intersphinx', -] +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.autosummary', + 'sphinx.ext.doctest', + 'sphinx.ext.extlinks', + 'sphinx.ext.todo', + 'numpydoc', # used to parse numpy-style docstrings for autodoc + 'IPython.sphinxext.ipython_console_highlighting', + 'IPython.sphinxext.ipython_directive', + 'sphinx.ext.intersphinx', + 'sphinx.ext.coverage', + 'sphinx.ext.ifconfig', + ] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/packages/pandas-gbq/docs/source/examples-tutorials.rst b/packages/pandas-gbq/docs/source/examples-tutorials.rst deleted file mode 100644 index bac945d559fe..000000000000 --- a/packages/pandas-gbq/docs/source/examples-tutorials.rst +++ /dev/null @@ -1,2 +0,0 @@ -Examples -======== diff --git a/packages/pandas-gbq/docs/source/index.rst b/packages/pandas-gbq/docs/source/index.rst index 01e07c480725..c0a2574dc700 100644 --- a/packages/pandas-gbq/docs/source/index.rst +++ b/packages/pandas-gbq/docs/source/index.rst @@ -6,13 +6,29 @@ Welcome to pandas-gbq's documentation! ====================================== +The :mod:`pandas_gbq` module provides a wrapper for Google's BigQuery +analytics web service to simplify retrieving results from BigQuery tables +using SQL-like queries. Result sets are parsed into a pandas +DataFrame with a shape and data types derived from the source table. +Additionally, DataFrames can be inserted into new BigQuery tables or appended +to existing tables. + +.. warning:: + + To use this module, you will need a valid BigQuery account. Refer to the + `BigQuery Documentation `__ + for details on the service itself. + Contents: .. toctree:: :maxdepth: 2 install.rst - examples-tutorials.rst + intro.rst + reading.rst + writing.rst + tables.rst changelog.rst diff --git a/packages/pandas-gbq/docs/source/install.rst b/packages/pandas-gbq/docs/source/install.rst index ea81060e61bb..2bcd8f112e46 100644 --- a/packages/pandas-gbq/docs/source/install.rst +++ b/packages/pandas-gbq/docs/source/install.rst @@ -23,5 +23,11 @@ Install from Source $ pip install git+https://github.com/pydata/pandas-gbq.git -Test ----- +Dependencies +------------ + +This module requires following additional dependencies: + +- `httplib2 `__: HTTP client +- `google-api-python-client `__: Google's API client +- `oauth2client `__: authentication and authorization for Google's API diff --git a/packages/pandas-gbq/docs/source/intro.rst b/packages/pandas-gbq/docs/source/intro.rst new file mode 100644 index 000000000000..0b40be0567e5 --- /dev/null +++ b/packages/pandas-gbq/docs/source/intro.rst @@ -0,0 +1,64 @@ +Introduction +============ + +Supported Data Types +++++++++++++++++++++ + +Pandas supports all these `BigQuery data types `__: +``STRING``, ``INTEGER`` (64bit), ``FLOAT`` (64 bit), ``BOOLEAN`` and +``TIMESTAMP`` (microsecond precision). Data types ``BYTES`` and ``RECORD`` +are not supported. + +Integer and boolean ``NA`` handling ++++++++++++++++++++++++++++++++++++ + +Since all columns in BigQuery queries are nullable, and NumPy lacks of ``NA`` +support for integer and boolean types, this module will store ``INTEGER`` or +``BOOLEAN`` columns with at least one ``NULL`` value as ``dtype=object``. +Otherwise those columns will be stored as ``dtype=int64`` or ``dtype=bool`` +respectively. + +This is opposite to default pandas behaviour which will promote integer +type to float in order to store NAs. +`See here for how this works in pandas `__ + +While this trade-off works well for most cases, it breaks down for storing +values greater than 2**53. Such values in BigQuery can represent identifiers +and unnoticed precision lost for identifier is what we want to avoid. + +.. _authentication: + +Authentication +'''''''''''''' + +Authentication to the Google ``BigQuery`` service is via ``OAuth 2.0``. +Is possible to authenticate with either user account credentials or service account credentials. + +Authenticating with user account credentials is as simple as following the prompts in a browser window +which will be automatically opened for you. You will be authenticated to the specified +``BigQuery`` account using the product name ``pandas GBQ``. It is only possible on local host. +The remote authentication using user account credentials is not currently supported in pandas. +Additional information on the authentication mechanism can be found +`here `__. + +Authentication with service account credentials is possible via the `'private_key'` parameter. This method +is particularly useful when working on remote servers (eg. jupyter iPython notebook on remote host). +Additional information on service accounts can be found +`here `__. + +Authentication via ``application default credentials`` is also possible. This is only valid +if the parameter ``private_key`` is not provided. This method also requires that +the credentials can be fetched from the environment the code is running in. +Otherwise, the OAuth2 client-side authentication is used. +Additional information on +`application default credentials `__. + +.. note:: + + The `'private_key'` parameter can be set to either the file path of the service account key + in JSON format, or key contents of the service account key in JSON format. + +.. note:: + + A private key can be obtained from the Google developers console by clicking + `here `__. Use JSON key type. diff --git a/packages/pandas-gbq/docs/source/reading.rst b/packages/pandas-gbq/docs/source/reading.rst new file mode 100644 index 000000000000..6b39cbd0fbd7 --- /dev/null +++ b/packages/pandas-gbq/docs/source/reading.rst @@ -0,0 +1,57 @@ +.. _reader: + +Reading +======= + +Suppose you want to load all data from an existing BigQuery table : `test_dataset.test_table` +into a DataFrame using the :func:`~read_gbq` function. + +.. code-block:: python + + # Insert your BigQuery Project ID Here + # Can be found in the Google web console + projectid = "xxxxxxxx" + + data_frame = read_gbq('SELECT * FROM test_dataset.test_table', projectid) + + +You can define which column from BigQuery to use as an index in the +destination DataFrame as well as a preferred column order as follows: + +.. code-block:: python + + data_frame = read_gbq('SELECT * FROM test_dataset.test_table', + index_col='index_column_name', + col_order=['col1', 'col2', 'col3'], projectid) + + +You can specify the query config as parameter to use additional options of your job. +For more information about query configuration parameters see +`here `__. + +.. code-block:: python + + configuration = { + 'query': { + "useQueryCache": False + } + } + data_frame = read_gbq('SELECT * FROM test_dataset.test_table', + configuration=configuration, projectid) + + +.. note:: + + You can find your project id in the `Google developers console `__. + + +.. note:: + + You can toggle the verbose output via the ``verbose`` flag which defaults to ``True``. + +.. note:: + + The ``dialect`` argument can be used to indicate whether to use BigQuery's ``'legacy'`` SQL + or BigQuery's ``'standard'`` SQL (beta). The default value is ``'legacy'``. For more information + on BigQuery's standard SQL, see `BigQuery SQL Reference + `__ diff --git a/packages/pandas-gbq/docs/source/tables.rst b/packages/pandas-gbq/docs/source/tables.rst new file mode 100644 index 000000000000..936ccb4ec52a --- /dev/null +++ b/packages/pandas-gbq/docs/source/tables.rst @@ -0,0 +1,22 @@ +.. _create_tables: + +Creating BigQuery Tables +======================== + +.. code-block:: ipython + + In [10]: gbq.generate_bq_schema(df, default_type='STRING') + + Out[10]: {'fields': [{'name': 'my_bool1', 'type': 'BOOLEAN'}, + {'name': 'my_bool2', 'type': 'BOOLEAN'}, + {'name': 'my_dates', 'type': 'TIMESTAMP'}, + {'name': 'my_float64', 'type': 'FLOAT'}, + {'name': 'my_int64', 'type': 'INTEGER'}, + {'name': 'my_string', 'type': 'STRING'}]} + +.. note:: + + If you delete and re-create a BigQuery table with the same name, but different table schema, + you must wait 2 minutes before streaming data into the table. As a workaround, consider creating + the new table with a different name. Refer to + `Google BigQuery issue 191 `__. diff --git a/packages/pandas-gbq/docs/source/writing.rst b/packages/pandas-gbq/docs/source/writing.rst new file mode 100644 index 000000000000..f0dc0aaa5182 --- /dev/null +++ b/packages/pandas-gbq/docs/source/writing.rst @@ -0,0 +1,91 @@ +.. _writer: + +Writing DataFrames +================== + +Assume we want to write a DataFrame ``df`` into a BigQuery table using :func:`~to_gbq`. + +.. ipython:: python + + import pandas as pd + df = pd.DataFrame({'my_string': list('abc'), + 'my_int64': list(range(1, 4)), + 'my_float64': np.arange(4.0, 7.0), + 'my_bool1': [True, False, True], + 'my_bool2': [False, True, False], + 'my_dates': pd.date_range('now', periods=3)}) + + df + df.dtypes + +.. code-block:: python + + to_gbq(df, 'my_dataset.my_table', projectid) + +.. note:: + + The destination table and destination dataset will automatically be created if they do not already exist. + +The ``if_exists`` argument can be used to dictate whether to ``'fail'``, ``'replace'`` +or ``'append'`` if the destination table already exists. The default value is ``'fail'``. + +For example, assume that ``if_exists`` is set to ``'fail'``. The following snippet will raise +a ``TableCreationError`` if the destination table already exists. + +.. code-block:: python + + to_gbq(df, 'my_dataset.my_table', projectid, if_exists='fail') + +.. note:: + + If the ``if_exists`` argument is set to ``'append'``, the destination dataframe will + be written to the table using the defined table schema and column types. The + dataframe must match the destination table in structure and data types. + If the ``if_exists`` argument is set to ``'replace'``, and the existing table has a + different schema, a delay of 2 minutes will be forced to ensure that the new schema + has propagated in the Google environment. See + `Google BigQuery issue 191 `__. + +Writing large DataFrames can result in errors due to size limitations being exceeded. +This can be avoided by setting the ``chunksize`` argument when calling :func:`~to_gbq`. +For example, the following writes ``df`` to a BigQuery table in batches of 10000 rows at a time: + +.. code-block:: python + + to_gbq(df, 'my_dataset.my_table', projectid, chunksize=10000) + +You can also see the progress of your post via the ``verbose`` flag which defaults to ``True``. +For example: + +.. code-block:: python + + In [8]: to_gbq(df, 'my_dataset.my_table', projectid, chunksize=10000, verbose=True) + + Streaming Insert is 10% Complete + Streaming Insert is 20% Complete + Streaming Insert is 30% Complete + Streaming Insert is 40% Complete + Streaming Insert is 50% Complete + Streaming Insert is 60% Complete + Streaming Insert is 70% Complete + Streaming Insert is 80% Complete + Streaming Insert is 90% Complete + Streaming Insert is 100% Complete + +.. note:: + + If an error occurs while streaming data to BigQuery, see + `Troubleshooting BigQuery Errors `__. + +.. note:: + + The BigQuery SQL query language has some oddities, see the + `BigQuery Query Reference Documentation `__. + +.. note:: + + While BigQuery uses SQL-like syntax, it has some important differences from traditional + databases both in functionality, API limitations (size and quantity of queries or uploads), + and how Google charges for use of the service. You should refer to `Google BigQuery documentation `__ + often as the service seems to be changing and evolving. BiqQuery is best for analyzing large + sets of data quickly, but it is not a direct replacement for a transactional database. From 255669daefd0c01b0db298e16567e3b6d4f97dcd Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Sun, 26 Feb 2017 16:33:46 -0500 Subject: [PATCH 035/519] add requirements-docs --- packages/pandas-gbq/docs/requirements-docs.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/pandas-gbq/docs/requirements-docs.txt b/packages/pandas-gbq/docs/requirements-docs.txt index 186b246693d9..e014c016cbb0 100644 --- a/packages/pandas-gbq/docs/requirements-docs.txt +++ b/packages/pandas-gbq/docs/requirements-docs.txt @@ -2,4 +2,3 @@ numpydoc sphinx sphinx_rtd_theme pandas -pandas-gbq From 035cba00cc9c18453e330bb5f617e90af97f7cbe Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Sun, 26 Feb 2017 16:36:48 -0500 Subject: [PATCH 036/519] add ipython to requirements-docs --- packages/pandas-gbq/docs/requirements-docs.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/pandas-gbq/docs/requirements-docs.txt b/packages/pandas-gbq/docs/requirements-docs.txt index e014c016cbb0..c224fb92c4c5 100644 --- a/packages/pandas-gbq/docs/requirements-docs.txt +++ b/packages/pandas-gbq/docs/requirements-docs.txt @@ -1,3 +1,4 @@ +ipython numpydoc sphinx sphinx_rtd_theme From 6e55c98f815ce5a4d11df5028e3fefc30db0e4f5 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Sun, 26 Feb 2017 16:38:40 -0500 Subject: [PATCH 037/519] add matlotlib to requirements-docs --- packages/pandas-gbq/docs/requirements-docs.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/pandas-gbq/docs/requirements-docs.txt b/packages/pandas-gbq/docs/requirements-docs.txt index c224fb92c4c5..afd31d061e10 100644 --- a/packages/pandas-gbq/docs/requirements-docs.txt +++ b/packages/pandas-gbq/docs/requirements-docs.txt @@ -1,4 +1,5 @@ ipython +matplotlib numpydoc sphinx sphinx_rtd_theme From 782428c41fc0333590b06e182c6cb977f432cfed Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Sun, 26 Feb 2017 17:37:58 -0500 Subject: [PATCH 038/519] add api and fix some links --- packages/pandas-gbq/docs/source/changelog.rst | 6 +++--- packages/pandas-gbq/docs/source/index.rst | 8 ++++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index b2a6b83e46d1..a2b004c9816f 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -4,7 +4,7 @@ Changelog 0.2.0 / 2017-03-?? ------------------ -- Bug with appending to a BigQuery table where fields have modes (NULLABLE,REQUIRED,REPEATED) specified. These modes were compared versus the remote schema and writing a table via ``to_gbq`` would previously raise. (:issue:`13`) +- Bug with appending to a BigQuery table where fields have modes (NULLABLE,REQUIRED,REPEATED) specified. These modes were compared versus the remote schema and writing a table via ``to_gbq`` would previously raise. :issue:`13` 0.1.2 / 2017-02-23 ------------------ @@ -13,5 +13,5 @@ Initial release of transfered code from `pandas `__ -- ``read_gbq`` now stores ``INTEGER`` columns as ``dtype=object`` if they contain ``NULL`` values. Otherwise they are stored as ``int64``. This prevents precision lost for integers greather than 2**53. Furthermore ``FLOAT`` columns with values above 10**4 are no longer casted to ``int64`` which also caused precision loss `here `__, and `here `__ +- ``read_gbq`` now allows query configuration preferences `pandas-GH#14742 `__ +- ``read_gbq`` now stores ``INTEGER`` columns as ``dtype=object`` if they contain ``NULL`` values. Otherwise they are stored as ``int64``. This prevents precision lost for integers greather than 2**53. Furthermore ``FLOAT`` columns with values above 10**4 are no longer casted to ``int64`` which also caused precision loss `pandas-GH#14064 `__, and `pandas-GH#14305 `__ diff --git a/packages/pandas-gbq/docs/source/index.rst b/packages/pandas-gbq/docs/source/index.rst index c0a2574dc700..c96d44e2611b 100644 --- a/packages/pandas-gbq/docs/source/index.rst +++ b/packages/pandas-gbq/docs/source/index.rst @@ -19,6 +19,14 @@ to existing tables. `BigQuery Documentation `__ for details on the service itself. +.. currentmodule:: pandas_gbq + +.. autosummary:: + :toctree: generated/ + + read_gbq + to_gbq + Contents: .. toctree:: From 3acdc64c9b7eb03e714a1905c1364d8d2a0017d5 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Sun, 26 Feb 2017 18:52:18 -0500 Subject: [PATCH 039/519] don't use generated in autosummary --- packages/pandas-gbq/docs/source/index.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/pandas-gbq/docs/source/index.rst b/packages/pandas-gbq/docs/source/index.rst index c96d44e2611b..2d4bc50cf86c 100644 --- a/packages/pandas-gbq/docs/source/index.rst +++ b/packages/pandas-gbq/docs/source/index.rst @@ -22,7 +22,6 @@ to existing tables. .. currentmodule:: pandas_gbq .. autosummary:: - :toctree: generated/ read_gbq to_gbq From cea499716b56715d43f24ac528a2e715c2361a0c Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 1 Mar 2017 21:39:55 -0500 Subject: [PATCH 040/519] DOC: add api.rst to docs, fix some typos --- packages/pandas-gbq/docs/source/api.rst | 14 ++++++++++++++ packages/pandas-gbq/docs/source/index.rst | 8 +------- packages/pandas-gbq/pandas_gbq/gbq.py | 14 +++++--------- 3 files changed, 20 insertions(+), 16 deletions(-) create mode 100644 packages/pandas-gbq/docs/source/api.rst diff --git a/packages/pandas-gbq/docs/source/api.rst b/packages/pandas-gbq/docs/source/api.rst new file mode 100644 index 000000000000..d1f50b9a8ac4 --- /dev/null +++ b/packages/pandas-gbq/docs/source/api.rst @@ -0,0 +1,14 @@ +.. currentmodule:: pandas_gbq +.. _api: + +************* +API Reference +************* + +.. autosummary:: + + read_gbq + to_gbq + +.. autofunction:: read_gbq +.. autofunction:: to_gbq diff --git a/packages/pandas-gbq/docs/source/index.rst b/packages/pandas-gbq/docs/source/index.rst index 2d4bc50cf86c..3e22b9723581 100644 --- a/packages/pandas-gbq/docs/source/index.rst +++ b/packages/pandas-gbq/docs/source/index.rst @@ -19,13 +19,6 @@ to existing tables. `BigQuery Documentation `__ for details on the service itself. -.. currentmodule:: pandas_gbq - -.. autosummary:: - - read_gbq - to_gbq - Contents: .. toctree:: @@ -36,6 +29,7 @@ Contents: reading.rst writing.rst tables.rst + api.rst changelog.rst diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 060724ed8de2..5b4ebad2608f 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -639,14 +639,12 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, **kwargs): r"""Load data from Google BigQuery. - THIS IS AN EXPERIMENTAL LIBRARY - The main method a user calls to execute a Query in Google BigQuery and read results into a pandas DataFrame. Google BigQuery API Client Library v2 for Python is used. - Documentation is available at - https://developers.google.com/api-client-library/python/apis/bigquery/v2 + Documentation is available `here + `__ Authentication to the Google BigQuery service is via OAuth 2.0. @@ -697,7 +695,7 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, configuration = {'query': {'useQueryCache': False}} For more information see `BigQuery SQL Reference - ` + `__ Returns ------- @@ -767,14 +765,12 @@ def to_gbq(dataframe, destination_table, project_id, chunksize=10000, verbose=True, reauth=False, if_exists='fail', private_key=None): """Write a DataFrame to a Google BigQuery table. - THIS IS AN EXPERIMENTAL LIBRARY - The main method a user calls to export pandas DataFrame contents to Google BigQuery table. Google BigQuery API Client Library v2 for Python is used. - Documentation is available at - https://developers.google.com/api-client-library/python/apis/bigquery/v2 + Documentation is available `here + `__ Authentication to the Google BigQuery service is via OAuth 2.0. From deac72c9163a63bc40b83f1d9bdb8ba64b441793 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Thu, 2 Mar 2017 09:36:14 -0500 Subject: [PATCH 041/519] make coverage check match reality :< --- packages/pandas-gbq/codecov.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/codecov.yml b/packages/pandas-gbq/codecov.yml index 59f8fdf6156f..e97acea7d26d 100644 --- a/packages/pandas-gbq/codecov.yml +++ b/packages/pandas-gbq/codecov.yml @@ -2,7 +2,7 @@ coverage: status: project: default: - target: '35' + target: '30' patch: default: target: '50' From fff526beb1b4aaf7ecd35b3a8586d2625658d56b Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Sat, 4 Mar 2017 11:37:22 -0500 Subject: [PATCH 042/519] update MANIFEST.in to include LICENSE.md --- packages/pandas-gbq/MANIFEST.in | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/packages/pandas-gbq/MANIFEST.in b/packages/pandas-gbq/MANIFEST.in index ec2ec72a9a4f..273fd19a980f 100644 --- a/packages/pandas-gbq/MANIFEST.in +++ b/packages/pandas-gbq/MANIFEST.in @@ -1,2 +1,18 @@ +include MANIFEST.in +include README.rst +include LICENSE.md +include setup.py + +graft pandas_gbq + +global-exclude *.so +global-exclude *.pyd +global-exclude *.pyc +global-exclude *~ +global-exclude \#* +global-exclude .git* +global-exclude .DS_Store +global-exclude *.png + include versioneer.py include pandas_gbq/_version.py From c62a613719ec78e0b1d6ad77b8aaffaf417e452c Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Sat, 4 Mar 2017 11:48:27 -0500 Subject: [PATCH 043/519] RLS: release 0.1.3 --- packages/pandas-gbq/docs/source/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index a2b004c9816f..1f16a1171337 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,7 +1,7 @@ Changelog ========= -0.2.0 / 2017-03-?? +0.1.3 / 2017-03-04 ------------------ - Bug with appending to a BigQuery table where fields have modes (NULLABLE,REQUIRED,REPEATED) specified. These modes were compared versus the remote schema and writing a table via ``to_gbq`` would previously raise. :issue:`13` From 9481c9e8923156ee85e117c18eb8145bb8ab336d Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Sat, 4 Mar 2017 12:35:14 -0500 Subject: [PATCH 044/519] only lint 3.6 build --- packages/pandas-gbq/.travis.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/pandas-gbq/.travis.yml b/packages/pandas-gbq/.travis.yml index e2fd48f67133..dabbd1d8352b 100644 --- a/packages/pandas-gbq/.travis.yml +++ b/packages/pandas-gbq/.travis.yml @@ -3,10 +3,10 @@ sudo: false language: python env: - - PYTHON=2.7 PANDAS=0.19.2 COVERAGE='false' - - PYTHON=3.4 PANDAS=0.18.1 COVERAGE='false' - - PYTHON=3.5 PANDAS=0.19.2 COVERAGE='true' - - PYTHON=3.6 PANDAS=0.19.2 COVERAGE='false' + - PYTHON=2.7 PANDAS=0.19.2 COVERAGE='false' LINT='false' + - PYTHON=3.4 PANDAS=0.18.1 COVERAGE='false' LINT='false' + - PYTHON=3.5 PANDAS=0.19.2 COVERAGE='true' LINT='false' + - PYTHON=3.6 PANDAS=0.19.2 COVERAGE='false' LINT='true' before_install: - echo "before_install" @@ -32,7 +32,5 @@ install: script: - pytest -s -v --cov=pandas_gbq --cov-report xml:/tmp/pytest-cov.xml pandas_gbq - - flake8 pandas_gbq -v - -after_success: - if [[ $COVERAGE == 'true' ]]; then codecov ; fi + - if [[ $LINT == 'true' ]]; flake8 pandas_gbq -v ; fi From 3c71b4116c5e8d5b84150bd4a1e3a80adc8d66fb Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Sat, 4 Mar 2017 12:45:01 -0500 Subject: [PATCH 045/519] typo in lint if --- packages/pandas-gbq/.travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/.travis.yml b/packages/pandas-gbq/.travis.yml index dabbd1d8352b..5a858c3f2297 100644 --- a/packages/pandas-gbq/.travis.yml +++ b/packages/pandas-gbq/.travis.yml @@ -33,4 +33,4 @@ install: script: - pytest -s -v --cov=pandas_gbq --cov-report xml:/tmp/pytest-cov.xml pandas_gbq - if [[ $COVERAGE == 'true' ]]; then codecov ; fi - - if [[ $LINT == 'true' ]]; flake8 pandas_gbq -v ; fi + - if [[ $LINT == 'true' ]]; then flake8 pandas_gbq -v ; fi From 33f2df968fc23871e01f56b89b1b305b8b0cb911 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Sat, 4 Mar 2017 14:23:03 -0500 Subject: [PATCH 046/519] Improve gbq.py code coverage (#19) --- packages/pandas-gbq/docs/source/changelog.rst | 5 + packages/pandas-gbq/pandas_gbq/gbq.py | 18 ++- .../pandas-gbq/pandas_gbq/tests/test_gbq.py | 128 ++++++++++++++++-- 3 files changed, 130 insertions(+), 21 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 1f16a1171337..2395be04a11f 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,6 +1,11 @@ Changelog ========= +0.1.4 / 2017-??-?? +------------------ + +- ``InvalidIndexColumn`` will be raised instead of ``InvalidColumnOrder`` in ``read_gbq`` when the index column specified does not exist in the BigQuery schema. :issue:`6` + 0.1.3 / 2017-03-04 ------------------ diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 5b4ebad2608f..c7d74504a21a 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -97,6 +97,15 @@ class InvalidColumnOrder(PandasError, ValueError): pass +class InvalidIndexColumn(PandasError, ValueError): + """ + Raised when the provided index column for output + results DataFrame does not match the schema + returned by BigQuery. + """ + pass + + class InvalidPageToken(PandasError, ValueError): """ Raised when Google BigQuery fails to return, @@ -308,9 +317,9 @@ def print_elapsed_seconds(self, prefix='Elapsed', postfix='s.', # http://stackoverflow.com/questions/1094841/reusable-library-to-get-human-readable-version-of-file-size @staticmethod - def sizeof_fmt(num, suffix='b'): + def sizeof_fmt(num, suffix='B'): fmt = "%3.1f %s%s" - for unit in ['', 'k', 'M', 'G', 'T', 'P', 'E', 'Z']: + for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']: if abs(num) < 1024.0: return fmt % (num, unit, suffix) num /= 1024.0 @@ -729,7 +738,7 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, if index_col in final_df.columns: final_df.set_index(index_col, inplace=True) else: - raise InvalidColumnOrder( + raise InvalidIndexColumn( 'Index column "{0}" does not exist in DataFrame.' .format(index_col) ) @@ -1049,9 +1058,6 @@ def datasets(self): dataset_response = list_dataset_response.get('datasets') next_page_token = list_dataset_response.get('nextPageToken') - if not dataset_response: - return dataset_list - for row_num, raw_row in enumerate(dataset_response): dataset_list.append( raw_row['datasetReference']['datasetId']) diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py index 6a3cad19f83b..b8b9f792c026 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -216,16 +216,29 @@ def _check_if_can_get_correct_default_credentials(): def clean_gbq_environment(dataset_prefix, private_key=None): dataset = gbq._Dataset(_get_project_id(), private_key=private_key) all_datasets = dataset.datasets() - for i in range(1, 10): - dataset_id = dataset_prefix + str(i) - if dataset_id in all_datasets: - table = gbq._Table(_get_project_id(), dataset_id, - private_key=private_key) - all_tables = dataset.tables(dataset_id) - for table_id in all_tables: - table.delete(table_id) - dataset.delete(dataset_id) + retry = 3 + while retry > 0: + try: + retry = retry - 1 + for i in range(1, 10): + dataset_id = dataset_prefix + str(i) + if dataset_id in all_datasets: + table = gbq._Table(_get_project_id(), dataset_id, + private_key=private_key) + all_tables = dataset.tables(dataset_id) + for table_id in all_tables: + table.delete(table_id) + + dataset.delete(dataset_id) + retry = 0 + except gbq.GenericGBQException as ex: + # Build in retry logic to work around the following error : + # An internal error occurred and the request could not be... + if 'An internal error occurred' in ex.message and retry > 0: + pass + else: + raise ex def make_mixed_dataframe_v2(test_size): @@ -519,7 +532,8 @@ def setUp(self): # - PER-TEST FIXTURES - # put here any instruction you want to be run *BEFORE* *EVERY* test is # executed. - pass + self.gbq_connector = gbq.GbqConnector( + _get_project_id(), private_key=_get_private_key_path()) @classmethod def tearDownClass(cls): @@ -714,6 +728,16 @@ def test_column_order(self): 'b'], 'string_3': ['c']})[col_order] tm.assert_frame_equal(result_frame, correct_frame) + def test_read_gbq_raises_invalid_column_order(self): + query = "SELECT 'a' AS string_1, 'b' AS string_2, 'c' AS string_3" + col_order = ['string_aaa', 'string_1', 'string_2'] + + # Column string_aaa does not exist. Should raise InvalidColumnOrder + with tm.assertRaises(gbq.InvalidColumnOrder): + gbq.read_gbq(query, project_id=_get_project_id(), + col_order=col_order, + private_key=_get_private_key_path()) + def test_column_order_plus_index(self): query = "SELECT 'a' AS string_1, 'b' AS string_2, 'c' AS string_3" col_order = ['string_3', 'string_2'] @@ -726,6 +750,16 @@ def test_column_order_plus_index(self): correct_frame = correct_frame[col_order] tm.assert_frame_equal(result_frame, correct_frame) + def test_read_gbq_raises_invalid_index_column(self): + query = "SELECT 'a' AS string_1, 'b' AS string_2, 'c' AS string_3" + col_order = ['string_3', 'string_2'] + + # Column string_bbb does not exist. Should raise InvalidIndexColumn + with tm.assertRaises(gbq.InvalidIndexColumn): + gbq.read_gbq(query, project_id=_get_project_id(), + index_col='string_bbb', col_order=col_order, + private_key=_get_private_key_path()) + def test_malformed_query(self): with tm.assertRaises(gbq.GenericGBQException): gbq.read_gbq("SELCET * FORM [publicdata:samples.shakespeare]", @@ -903,6 +937,41 @@ def test_configuration_without_query(self): private_key=_get_private_key_path(), configuration=config) + def test_configuration_raises_value_error_with_multiple_config(self): + sql_statement = 'SELECT 1' + config = { + 'query': { + "query": sql_statement, + "useQueryCache": False, + }, + 'load': { + "query": sql_statement, + "useQueryCache": False, + } + } + # Test that only ValueError is raised with multiple configurations + with tm.assertRaises(ValueError): + gbq.read_gbq(sql_statement, project_id=_get_project_id(), + private_key=_get_private_key_path(), + configuration=config) + + def test_query_response_bytes(self): + self.assertEqual(self.gbq_connector.sizeof_fmt(999), "999.0 B") + self.assertEqual(self.gbq_connector.sizeof_fmt(1024), "1.0 KB") + self.assertEqual(self.gbq_connector.sizeof_fmt(1099), "1.1 KB") + self.assertEqual(self.gbq_connector.sizeof_fmt(1044480), "1020.0 KB") + self.assertEqual(self.gbq_connector.sizeof_fmt(1048576), "1.0 MB") + self.assertEqual(self.gbq_connector.sizeof_fmt(1048576000), + "1000.0 MB") + self.assertEqual(self.gbq_connector.sizeof_fmt(1073741824), "1.0 GB") + self.assertEqual(self.gbq_connector.sizeof_fmt(1.099512E12), "1.0 TB") + self.assertEqual(self.gbq_connector.sizeof_fmt(1.125900E15), "1.0 PB") + self.assertEqual(self.gbq_connector.sizeof_fmt(1.152922E18), "1.0 EB") + self.assertEqual(self.gbq_connector.sizeof_fmt(1.180592E21), "1.0 ZB") + self.assertEqual(self.gbq_connector.sizeof_fmt(1.208926E24), "1.0 YB") + self.assertEqual(self.gbq_connector.sizeof_fmt(1.208926E28), + "10000.0 YB") + class TestToGBQIntegrationWithServiceAccountKeyPath(tm.TestCase): # Changes to BigQuery table schema may take up to 2 minutes as of May 2015 @@ -1035,6 +1104,16 @@ def test_upload_data_if_table_exists_replace(self): private_key=_get_private_key_path()) self.assertEqual(result['num_rows'][0], 5) + def test_upload_data_if_table_exists_raises_value_error(self): + test_id = "4" + test_size = 10 + df = make_mixed_dataframe_v2(test_size) + + # Test invalid value for if_exists parameter raises value error + with tm.assertRaises(ValueError): + gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), + if_exists='xxxxx', private_key=_get_private_key_path()) + def test_google_upload_errors_should_raise_exception(self): raise pytest.skip("buggy test") @@ -1062,14 +1141,18 @@ def test_generate_schema(self): def test_create_table(self): test_id = "6" - test_schema = {'fields': [{'name': 'A', 'type': 'FLOAT'}, - {'name': 'B', 'type': 'FLOAT'}, - {'name': 'C', 'type': 'STRING'}, - {'name': 'D', 'type': 'TIMESTAMP'}]} - self.table.create(TABLE_ID + test_id, test_schema) + schema = gbq._generate_bq_schema(tm.makeMixedDataFrame()) + self.table.create(TABLE_ID + test_id, schema) self.assertTrue(self.table.exists(TABLE_ID + test_id), 'Expected table to exist') + def test_create_table_already_exists(self): + test_id = "6" + schema = gbq._generate_bq_schema(tm.makeMixedDataFrame()) + self.table.create(TABLE_ID + test_id, schema) + with tm.assertRaises(gbq.TableCreationError): + self.table.create(TABLE_ID + test_id, schema) + def test_table_does_not_exist(self): test_id = "7" self.assertTrue(not self.table.exists(TABLE_ID + test_id), @@ -1086,6 +1169,10 @@ def test_delete_table(self): self.assertTrue(not self.table.exists( TABLE_ID + test_id), 'Expected table not to exist') + def test_delete_table_not_found(self): + with tm.assertRaises(gbq.NotFoundException): + self.table.delete(TABLE_ID + "not_found") + def test_list_table(self): test_id = "9" test_schema = {'fields': [{'name': 'A', 'type': 'FLOAT'}, @@ -1210,6 +1297,12 @@ def test_create_dataset(self): self.assertTrue(dataset_id in self.dataset.datasets(), 'Expected dataset to exist') + def test_create_dataset_already_exists(self): + dataset_id = self.dataset_prefix + "3" + self.dataset.create(dataset_id) + with tm.assertRaises(gbq.DatasetCreationError): + self.dataset.create(dataset_id) + def test_delete_dataset(self): dataset_id = self.dataset_prefix + "4" self.dataset.create(dataset_id) @@ -1217,6 +1310,11 @@ def test_delete_dataset(self): self.assertTrue(dataset_id not in self.dataset.datasets(), 'Expected dataset not to exist') + def test_delete_dataset_not_found(self): + dataset_id = self.dataset_prefix + "not_found" + with tm.assertRaises(gbq.NotFoundException): + self.dataset.delete(dataset_id) + def test_dataset_exists(self): dataset_id = self.dataset_prefix + "5" self.dataset.create(dataset_id) From ba5e35ce9e790bc786289e2002213e639582a25c Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Sat, 4 Mar 2017 15:16:42 -0500 Subject: [PATCH 047/519] coverage badge --- packages/pandas-gbq/README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/pandas-gbq/README.rst b/packages/pandas-gbq/README.rst index 2cd43725f13e..fd9972464cd4 100644 --- a/packages/pandas-gbq/README.rst +++ b/packages/pandas-gbq/README.rst @@ -2,6 +2,8 @@ :target: https://travis-ci.org/pydata/pandas-gbq .. |Version Status| image:: https://img.shields.io/pypi/v/pandas-gbq.svg :target: https://pypi.python.org/pypi/pandas-gbq/ +.. |Coverage Status| image:: https://img.shields.io/codecov/c/github/pydata/pandas-gbq.svg + :target: https://codecov.io/gh/pydata/pandas-gbq/ pandas-gbq ========== From 8a0beac0e6f161a4d6f7487d145ac6a933c8a7e6 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Sat, 4 Mar 2017 15:18:01 -0500 Subject: [PATCH 048/519] actually show the coverage badge --- packages/pandas-gbq/README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/README.rst b/packages/pandas-gbq/README.rst index fd9972464cd4..dc1ac1566274 100644 --- a/packages/pandas-gbq/README.rst +++ b/packages/pandas-gbq/README.rst @@ -8,7 +8,7 @@ pandas-gbq ========== -|Build Status| |Version Status| +|Build Status| |Version Status| |Coverage Status| **pandas-gbq** is a package providing an interface to the Google Big Query interface from pandas From e845bbba76b4027b7de751d97cffc3e5c7192a0c Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Mon, 6 Mar 2017 09:32:14 -0500 Subject: [PATCH 049/519] add conda-forge instructions in install --- packages/pandas-gbq/README.rst | 7 +++++++ packages/pandas-gbq/docs/source/install.rst | 13 +++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/README.rst b/packages/pandas-gbq/README.rst index dc1ac1566274..bf151310de52 100644 --- a/packages/pandas-gbq/README.rst +++ b/packages/pandas-gbq/README.rst @@ -17,6 +17,13 @@ Installation ------------ +Install latest release version via conda +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: shell + + $ conda install pandas-gbq --channel conda-forge + Install latest release version via pip ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/packages/pandas-gbq/docs/source/install.rst b/packages/pandas-gbq/docs/source/install.rst index 2bcd8f112e46..a78823c2af70 100644 --- a/packages/pandas-gbq/docs/source/install.rst +++ b/packages/pandas-gbq/docs/source/install.rst @@ -1,7 +1,16 @@ Install pandas-gbq ================== -You can install pandas-gbq with ``pip``, or by installing from source. +You can install pandas-gbq with ``conda``, ``pip``, or by installing from source. + +Conda +~~~~~ + +.. code-block:: shell + + $ conda install pandas-gbq --channel conda-forge + +This installs pandas-gbq and all common dependencies, including ``pandas``. Pip --- @@ -12,7 +21,7 @@ To install the latest version of pandas-gbq: from the $ pip install pandas-gbq -U -This installs pandas-gbq and all common dependencies, including Pandas. +This installs pandas-gbq and all common dependencies, including ``pandas``. Install from Source From b8f93a463450c22f9ecde71168eab0e2a1dc9ca9 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Mon, 6 Mar 2017 15:48:05 -0500 Subject: [PATCH 050/519] install doc formatting --- packages/pandas-gbq/docs/source/install.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/pandas-gbq/docs/source/install.rst b/packages/pandas-gbq/docs/source/install.rst index a78823c2af70..5b0490f87ef1 100644 --- a/packages/pandas-gbq/docs/source/install.rst +++ b/packages/pandas-gbq/docs/source/install.rst @@ -1,10 +1,10 @@ -Install pandas-gbq -================== +Installation +============ You can install pandas-gbq with ``conda``, ``pip``, or by installing from source. Conda -~~~~~ +----- .. code-block:: shell From 2e6901df1c148018375a2c2d32d61666173d782a Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 7 Mar 2017 09:38:29 -0500 Subject: [PATCH 051/519] DOC: Add contributor guidelines (#21) --- .../pandas-gbq/docs/source/contributing.rst | 441 ++++++++++++++++++ 1 file changed, 441 insertions(+) create mode 100644 packages/pandas-gbq/docs/source/contributing.rst diff --git a/packages/pandas-gbq/docs/source/contributing.rst b/packages/pandas-gbq/docs/source/contributing.rst new file mode 100644 index 000000000000..c9de36e29d3c --- /dev/null +++ b/packages/pandas-gbq/docs/source/contributing.rst @@ -0,0 +1,441 @@ +.. _contributing: + +************************** +Contributing to pandas-gbq +************************** + +.. contents:: Table of contents: + :local: + +Where to start? +=============== + +All contributions, bug reports, bug fixes, documentation improvements, +enhancements and ideas are welcome. + +If you are simply looking to start working with the *pandas-gbq* codebase, navigate to the +`GitHub "issues" tab `_ and start looking through +interesting issues. + +Or maybe through using *pandas-gbq* you have an idea of your own or are looking for something +in the documentation and thinking 'this can be improved'...you can do something about it! + +Feel free to ask questions on the `mailing list +`_. + +Bug reports and enhancement requests +==================================== + +Bug reports are an important part of making *pandas-gbq* more stable. Having a complete bug report +will allow others to reproduce the bug and provide insight into fixing it. Because many versions of +*pandas-gbq* are supported, knowing version information will also identify improvements made since +previous versions. Trying the bug-producing code out on the *master* branch is often a worthwhile exercise +to confirm the bug still exists. It is also worth searching existing bug reports and pull requests +to see if the issue has already been reported and/or fixed. + +Bug reports must: + +#. Include a short, self-contained Python snippet reproducing the problem. + You can format the code nicely by using `GitHub Flavored Markdown + `_:: + + ```python + >>> from pandas_gbq import gbq + >>> df = gbq.read_gbq(...) + ... + ``` + +#. Include the full version string of *pandas-gbq*. :: + + ```python + >>> import pandas_gbq + >>> pandas_gbq.__version__ + ... + ``` + +#. Explain why the current behavior is wrong/not desired and what you expect instead. + +The issue will then show up to the *pandas-gbq* community and be open to comments/ideas from others. + +Working with the code +===================== + +Now that you have an issue you want to fix, enhancement to add, or documentation to improve, +you need to learn how to work with GitHub and the *pandas-gbq* code base. + +Version control, Git, and GitHub +-------------------------------- + +To the new user, working with Git is one of the more daunting aspects of contributing to *pandas-gbq*. +It can very quickly become overwhelming, but sticking to the guidelines below will help keep the process +straightforward and mostly trouble free. As always, if you are having difficulties please +feel free to ask for help. + +The code is hosted on `GitHub `_. To +contribute you will need to sign up for a `free GitHub account +`_. We use `Git `_ for +version control to allow many people to work together on the project. + +Some great resources for learning Git: + +* the `GitHub help pages `_. +* the `NumPy's documentation `_. +* Matthew Brett's `Pydagogue `_. + +Getting started with Git +------------------------ + +`GitHub has instructions `__ for installing git, +setting up your SSH key, and configuring git. All these steps need to be completed before +you can work seamlessly between your local repository and GitHub. + +.. _contributing.forking: + +Forking +------- + +You will need your own fork to work on the code. Go to the `pandas-gbq project +page `_ and hit the ``Fork`` button. You will +want to clone your fork to your machine:: + + git clone git@github.com:your-user-name/pandas-gbq.git pandas-gbq-yourname + cd pandas-gbq-yourname + git remote add upstream git://github.com/pydata/pandas-gbq.git + +This creates the directory `pandas-gbq-yourname` and connects your repository to +the upstream (main project) *pandas-gbq* repository. + +The testing suite will run automatically on Travis-CI once your pull request is submitted. +However, if you wish to run the test suite on a branch prior to submitting the pull request, +then Travis-CI needs to be hooked up to your GitHub repository. Instructions for doing so +are `here `__. + +Creating a branch +----------------- + +You want your master branch to reflect only production-ready code, so create a +feature branch for making your changes. For example:: + + git branch shiny-new-feature + git checkout shiny-new-feature + +The above can be simplified to:: + + git checkout -b shiny-new-feature + +This changes your working directory to the shiny-new-feature branch. Keep any +changes in this branch specific to one bug or feature so it is clear +what the branch brings to *pandas-gbq*. You can have many shiny-new-features +and switch in between them using the git checkout command. + +To update this branch, you need to retrieve the changes from the master branch:: + + git fetch upstream + git rebase upstream/master + +This will replay your commits on top of the latest pandas-gbq git master. If this +leads to merge conflicts, you must resolve these before submitting your pull +request. If you have uncommitted changes, you will need to ``stash`` them prior +to updating. This will effectively store your changes and they can be reapplied +after updating. + +Contributing to the code base +============================= + +.. contents:: Code Base: + :local: + +Code standards +-------------- + +Writing good code is not just about what you write. It is also about *how* you +write it. During testing on Travis-CI, several tools will be run to check your +code for stylistic errors. Generating any warnings will cause the test to fail. +Thus, good style is a requirement for submitting code to *pandas-gbq*. + +In addition, because a lot of people use our library, it is important that we +do not make sudden changes to the code that could have the potential to break +a lot of user code as a result, that is, we need it to be as *backwards compatible* +as possible to avoid mass breakages. + +Python (PEP8) +~~~~~~~~~~~~~ + +*pandas-gbq* uses the `PEP8 `_ standard. +There are several tools to ensure you abide by this standard. Here are *some* of +the more common ``PEP8`` issues: + + - we restrict line-length to 79 characters to promote readability + - passing arguments should have spaces after commas, e.g. ``foo(arg1, arg2, kw1='bar')`` + +Travis-CI will run the `flake8 `_ tool +and report any stylistic errors in your code. Therefore, it is helpful before +submitting code to run the check yourself on the diff:: + + git diff master | flake8 --diff + +Backwards Compatibility +~~~~~~~~~~~~~~~~~~~~~~~ + +Please try to maintain backward compatibility. If you think breakage is required, +clearly state why as part of the pull request. Also, be careful when changing method +signatures and add deprecation warnings where needed. + +Test-driven development/code writing +------------------------------------ + +*pandas-gbq* is serious about testing and strongly encourages contributors to embrace +`test-driven development (TDD) `_. +This development process "relies on the repetition of a very short development cycle: +first the developer writes an (initially failing) automated test case that defines a desired +improvement or new function, then produces the minimum amount of code to pass that test." +So, before actually writing any code, you should write your tests. Often the test can be +taken from the original GitHub issue. However, it is always worth considering additional +use cases and writing corresponding tests. + +Adding tests is one of the most common requests after code is pushed to *pandas-gbq*. Therefore, +it is worth getting in the habit of writing tests ahead of time so this is never an issue. + +Like many packages, *pandas-gbq* uses `pytest `_. + +Running the test suite +~~~~~~~~~~~~~~~~~~~~~~ + +The tests can then be run directly inside your Git clone (without having to +install *pandas-gbq*) by typing:: + + pytest pandas_gbq + +The tests suite is exhaustive and takes around 20 minutes to run. Often it is +worth running only a subset of tests first around your changes before running the +entire suite. + +The easiest way to do this is with:: + + pytest pandas_gbq/path/to/test.py -k regex_matching_test_name + +Or with one of the following constructs:: + + pytest pandas_gbq/tests/[test-module].py + pytest pandas_gbq/tests/[test-module].py::[TestClass] + pytest pandas_gbq/tests/[test-module].py::[TestClass]::[test_method] + +For more, see the `pytest `_ documentation. + +.. _contributing.gbq_integration_tests: + +Running Google BigQuery Integration Tests +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You will need to create a Google BigQuery private key in JSON format in +order to run Google BigQuery integration tests on your local machine and +on Travis-CI. The first step is to create a `service account +`__. + +Integration tests are skipped in pull requests because the credentials that +are required for running Google BigQuery integration tests are +`encrypted `__ +on Travis-CI and are only accessible from the pydata/pandas-gbq repository. The +credentials won't be available on forks of pandas-gbq. Here are the steps to run +gbq integration tests on a forked repository: + +#. Go to `Travis CI `__ and sign in with your GitHub + account. +#. Click on the ``+`` icon next to the ``My Repositories`` list and enable + Travis builds for your fork. +#. Click on the gear icon to edit your travis build, and add two environment + variables: + + - ``GBQ_PROJECT_ID`` with the value being the ID of your BigQuery project. + + - ``SERVICE_ACCOUNT_KEY`` with the value being the contents of the JSON key + that you downloaded for your service account. Use single quotes around + your JSON key to ensure that it is treated as a string. + + For both environment variables, keep the "Display value in build log" option + DISABLED. These variables contain sensitive data and you do not want their + contents being exposed in build logs. +#. Your branch should be tested automatically once it is pushed. You can check + the status by visiting your Travis branches page which exists at the + following location: https://travis-ci.org/your-user-name/pandas-gbq/branches . + Click on a build job for your branch. + +Documenting your code +--------------------- + +Changes should be reflected in the release notes located in ``doc/source/changelog.rst``. +This file contains an ongoing change log. Add an entry to this file to document your fix, +enhancement or (unavoidable) breaking change. Make sure to include the GitHub issue number +when adding your entry (using `` :issue:`1234` `` where `1234` is the issue/pull request number). + +If your code is an enhancement, it is most likely necessary to add usage +examples to the existing documentation. Further, to let users know when +this feature was added, the ``versionadded`` directive is used. The sphinx +syntax for that is: + +.. code-block:: rst + + .. versionadded:: 0.1.3 + +This will put the text *New in version 0.1.3* wherever you put the sphinx +directive. This should also be put in the docstring when adding a new function +or method. + +Contributing your changes to *pandas-gbq* +===================================== + +Committing your code +-------------------- + +Keep style fixes to a separate commit to make your pull request more readable. + +Once you've made changes, you can see them by typing:: + + git status + +If you have created a new file, it is not being tracked by git. Add it by typing:: + + git add path/to/file-to-be-added.py + +Doing 'git status' again should give something like:: + + # On branch shiny-new-feature + # + # modified: /relative/path/to/file-you-added.py + # + +Finally, commit your changes to your local repository with an explanatory message. *pandas-gbq* +uses a convention for commit message prefixes and layout. Here are +some common prefixes along with general guidelines for when to use them: + + * ENH: Enhancement, new functionality + * BUG: Bug fix + * DOC: Additions/updates to documentation + * TST: Additions/updates to tests + * BLD: Updates to the build process/scripts + * PERF: Performance improvement + * CLN: Code cleanup + +The following defines how a commit message should be structured. Please reference the +relevant GitHub issues in your commit message using GH1234 or #1234. Either style +is fine, but the former is generally preferred: + + * a subject line with `< 80` chars. + * One blank line. + * Optionally, a commit message body. + +Now you can commit your changes in your local repository:: + + git commit -m + +Combining commits +----------------- + +If you have multiple commits, you may want to combine them into one commit, often +referred to as "squashing" or "rebasing". This is a common request by package maintainers +when submitting a pull request as it maintains a more compact commit history. To rebase +your commits:: + + git rebase -i HEAD~# + +Where # is the number of commits you want to combine. Then you can pick the relevant +commit message and discard others. + +To squash to the master branch do:: + + git rebase -i master + +Use the ``s`` option on a commit to ``squash``, meaning to keep the commit messages, +or ``f`` to ``fixup``, meaning to merge the commit messages. + +Then you will need to push the branch (see below) forcefully to replace the current +commits with the new ones:: + + git push origin shiny-new-feature -f + + +Pushing your changes +-------------------- + +When you want your changes to appear publicly on your GitHub page, push your +forked feature branch's commits:: + + git push origin shiny-new-feature + +Here ``origin`` is the default name given to your remote repository on GitHub. +You can see the remote repositories:: + + git remote -v + +If you added the upstream repository as described above you will see something +like:: + + origin git@github.com:yourname/pandas-gbq.git (fetch) + origin git@github.com:yourname/pandas-gbq.git (push) + upstream git://github.com/pydata/pandas-gbq.git (fetch) + upstream git://github.com/pydata/pandas-gbq.git (push) + +Now your code is on GitHub, but it is not yet a part of the *pandas-gbq* project. For that to +happen, a pull request needs to be submitted on GitHub. + +Review your code +---------------- + +When you're ready to ask for a code review, file a pull request. Before you do, once +again make sure that you have followed all the guidelines outlined in this document +regarding code style, tests, performance tests, and documentation. You should also +double check your branch changes against the branch it was based on: + +#. Navigate to your repository on GitHub -- https://github.com/your-user-name/pandas-gbq +#. Click on ``Branches`` +#. Click on the ``Compare`` button for your feature branch +#. Select the ``base`` and ``compare`` branches, if necessary. This will be ``master`` and + ``shiny-new-feature``, respectively. + +Finally, make the pull request +------------------------------ + +If everything looks good, you are ready to make a pull request. A pull request is how +code from a local repository becomes available to the GitHub community and can be looked +at and eventually merged into the master version. This pull request and its associated +changes will eventually be committed to the master branch and available in the next +release. To submit a pull request: + +#. Navigate to your repository on GitHub +#. Click on the ``Pull Request`` button +#. You can then click on ``Commits`` and ``Files Changed`` to make sure everything looks + okay one last time +#. Write a description of your changes in the ``Preview Discussion`` tab +#. Click ``Send Pull Request``. + +This request then goes to the repository maintainers, and they will review +the code. If you need to make more changes, you can make them in +your branch, push them to GitHub, and the pull request will be automatically +updated. Pushing them to GitHub again is done by:: + + git push -f origin shiny-new-feature + +This will automatically update your pull request with the latest code and restart the +Travis-CI tests. + +Delete your merged branch (optional) +------------------------------------ + +Once your feature branch is accepted into upstream, you'll probably want to get rid of +the branch. First, merge upstream master into your branch so git knows it is safe to +delete your branch:: + + git fetch upstream + git checkout master + git merge upstream/master + +Then you can just do:: + + git branch -d shiny-new-feature + +Make sure you use a lower-case ``-d``, or else git won't warn you if your feature +branch has not actually been merged. + +The branch will still exist on GitHub, so to delete it there do:: + + git push origin --delete shiny-new-feature \ No newline at end of file From 09d32f175a945e648e9e279d7ce738f9f73c1ca5 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Tue, 7 Mar 2017 09:45:09 -0500 Subject: [PATCH 052/519] add contributing.rst to index.rst --- packages/pandas-gbq/docs/source/contributing.rst | 16 ++++++++-------- packages/pandas-gbq/docs/source/index.rst | 1 + packages/pandas-gbq/docs/source/reading.rst | 4 ++-- packages/pandas-gbq/docs/source/tables.rst | 4 ++-- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/packages/pandas-gbq/docs/source/contributing.rst b/packages/pandas-gbq/docs/source/contributing.rst index c9de36e29d3c..f0cb5bdba6be 100644 --- a/packages/pandas-gbq/docs/source/contributing.rst +++ b/packages/pandas-gbq/docs/source/contributing.rst @@ -37,21 +37,21 @@ Bug reports must: #. Include a short, self-contained Python snippet reproducing the problem. You can format the code nicely by using `GitHub Flavored Markdown - `_:: + `__ + + .. code-block:: python - ```python >>> from pandas_gbq import gbq >>> df = gbq.read_gbq(...) ... - ``` -#. Include the full version string of *pandas-gbq*. :: +#. Include the full version string of *pandas-gbq*. + + .. code-block:: python - ```python >>> import pandas_gbq >>> pandas_gbq.__version__ ... - ``` #. Explain why the current behavior is wrong/not desired and what you expect instead. @@ -282,7 +282,7 @@ directive. This should also be put in the docstring when adding a new function or method. Contributing your changes to *pandas-gbq* -===================================== +========================================= Committing your code -------------------- @@ -438,4 +438,4 @@ branch has not actually been merged. The branch will still exist on GitHub, so to delete it there do:: - git push origin --delete shiny-new-feature \ No newline at end of file + git push origin --delete shiny-new-feature diff --git a/packages/pandas-gbq/docs/source/index.rst b/packages/pandas-gbq/docs/source/index.rst index 3e22b9723581..44a61843697c 100644 --- a/packages/pandas-gbq/docs/source/index.rst +++ b/packages/pandas-gbq/docs/source/index.rst @@ -30,6 +30,7 @@ Contents: writing.rst tables.rst api.rst + contributing.rst changelog.rst diff --git a/packages/pandas-gbq/docs/source/reading.rst b/packages/pandas-gbq/docs/source/reading.rst index 6b39cbd0fbd7..ebfcfa9ba345 100644 --- a/packages/pandas-gbq/docs/source/reading.rst +++ b/packages/pandas-gbq/docs/source/reading.rst @@ -1,7 +1,7 @@ .. _reader: -Reading -======= +Reading Tables +============== Suppose you want to load all data from an existing BigQuery table : `test_dataset.test_table` into a DataFrame using the :func:`~read_gbq` function. diff --git a/packages/pandas-gbq/docs/source/tables.rst b/packages/pandas-gbq/docs/source/tables.rst index 936ccb4ec52a..ba07125d43da 100644 --- a/packages/pandas-gbq/docs/source/tables.rst +++ b/packages/pandas-gbq/docs/source/tables.rst @@ -1,7 +1,7 @@ .. _create_tables: -Creating BigQuery Tables -======================== +Creating Tables +=============== .. code-block:: ipython From c4be3b6d29717e6969eafcdb2f8b4752104a4af2 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Fri, 17 Mar 2017 22:13:56 -0400 Subject: [PATCH 053/519] clean up imports ; remove deprecated pandas.tools.merge.concat import --- packages/pandas-gbq/pandas_gbq/gbq.py | 4 +--- packages/pandas-gbq/pandas_gbq/tests/test_gbq.py | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index c7d74504a21a..ee42ab083097 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -9,9 +9,7 @@ import numpy as np from distutils.version import StrictVersion -from pandas import compat -from pandas.core.api import DataFrame -from pandas.tools.merge import concat +from pandas import compat, DataFrame, concat from pandas.core.common import PandasError from pandas.compat import lzip, bytes_to_str diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py index b8b9f792c026..338d9b1eb33b 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -13,9 +13,8 @@ from distutils.version import StrictVersion from pandas import compat -from pandas import NaT from pandas.compat import u, range -from pandas.core.frame import DataFrame +from pandas import NaT, DataFrame from pandas_gbq import gbq import pandas.util.testing as tm from pandas.compat.numpy import np_datetime64_compat From 281064d009584c18f1c3d37324b4f471498c1a1e Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Fri, 17 Mar 2017 22:15:01 -0400 Subject: [PATCH 054/519] DOC: 0.1.4 changelog DOC: update release-procedure.md --- packages/pandas-gbq/docs/source/changelog.rst | 2 +- packages/pandas-gbq/release-procedure.md | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 2395be04a11f..401146935017 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,7 +1,7 @@ Changelog ========= -0.1.4 / 2017-??-?? +0.1.4 / 2017-03-17 ------------------ - ``InvalidIndexColumn`` will be raised instead of ``InvalidColumnOrder`` in ``read_gbq`` when the index column specified does not exist in the BigQuery schema. :issue:`6` diff --git a/packages/pandas-gbq/release-procedure.md b/packages/pandas-gbq/release-procedure.md index e6adee983f08..c0bc1ab22078 100644 --- a/packages/pandas-gbq/release-procedure.md +++ b/packages/pandas-gbq/release-procedure.md @@ -12,8 +12,7 @@ python setup.py register sdist bdist_wheel --universal twine upload dist/* -* Update anaconda recipe. +* Do a pull-request to the feedstock on `pandas-gbq-feedstock `__ - This should happen automatically within a day or two. - -* Update conda recipe feedstock on `conda-forge `_. + update the version + update the SHA256 (retrieve from PyPI) From 319945271b379efaccfa9715902ed1a1d7aa5c7b Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Sat, 18 Mar 2017 10:15:07 -0400 Subject: [PATCH 055/519] DOC: fix README.rst to dispaly correctly --- packages/pandas-gbq/README.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/pandas-gbq/README.rst b/packages/pandas-gbq/README.rst index bf151310de52..803ad7dffeac 100644 --- a/packages/pandas-gbq/README.rst +++ b/packages/pandas-gbq/README.rst @@ -1,10 +1,3 @@ -.. |Build Status| image:: https://travis-ci.org/pydata/pandas-gbq.svg?branch=master - :target: https://travis-ci.org/pydata/pandas-gbq -.. |Version Status| image:: https://img.shields.io/pypi/v/pandas-gbq.svg - :target: https://pypi.python.org/pypi/pandas-gbq/ -.. |Coverage Status| image:: https://img.shields.io/codecov/c/github/pydata/pandas-gbq.svg - :target: https://codecov.io/gh/pydata/pandas-gbq/ - pandas-gbq ========== @@ -43,3 +36,10 @@ Usage ----- See the `pandas-gbq documentation `_ for more details. + +.. |Build Status| image:: https://travis-ci.org/pydata/pandas-gbq.svg?branch=master + :target: https://travis-ci.org/pydata/pandas-gbq +.. |Version Status| image:: https://img.shields.io/pypi/v/pandas-gbq.svg + :target: https://pypi.python.org/pypi/pandas-gbq/ +.. |Coverage Status| image:: https://img.shields.io/codecov/c/github/pydata/pandas-gbq.svg + :target: https://codecov.io/gh/pydata/pandas-gbq/ From 569d90ed9b29cd8eac46c35ad8e202f67a47d48a Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 20 Apr 2017 13:21:44 -0700 Subject: [PATCH 056/519] DOC: Link to contributing guide (#28) Full contributing (and testing) instructions are included in the docs. http://pandas-gbq.readthedocs.io/en/latest/contributing.html I didn't find these docs (GH27) because I didn't expect them to be in the user documentation, and I'm used to looking for a CONTRIBUTING file. By creating a CONTRIBUTING.md file, GitHub users will be more likely to find them. See: https://github.com/blog/1184-contributing-guidelines --- packages/pandas-gbq/CONTRIBUTING.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 packages/pandas-gbq/CONTRIBUTING.md diff --git a/packages/pandas-gbq/CONTRIBUTING.md b/packages/pandas-gbq/CONTRIBUTING.md new file mode 100644 index 000000000000..e67468732a93 --- /dev/null +++ b/packages/pandas-gbq/CONTRIBUTING.md @@ -0,0 +1,5 @@ +# Contributing + +See the [contributing guide in the pandas-gbq +docs](http://pandas-gbq.readthedocs.io/en/latest/contributing.html). + From 93f8e30ef59ed556048a0b4a7a3bed84fd68d65b Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 20 Apr 2017 18:31:16 -0700 Subject: [PATCH 057/519] TST: Load integration test creds from env vars (#29) Load the credentials for integration tests from environment variables. This will make it easier to run the integration tests locally. Also adds instructions for running the integrations tests locally. Closes GH#27. --- .../pandas-gbq/docs/source/contributing.rst | 13 ++++++++--- .../pandas-gbq/pandas_gbq/tests/test_gbq.py | 22 +++++++------------ 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/packages/pandas-gbq/docs/source/contributing.rst b/packages/pandas-gbq/docs/source/contributing.rst index f0cb5bdba6be..d3376a30cb08 100644 --- a/packages/pandas-gbq/docs/source/contributing.rst +++ b/packages/pandas-gbq/docs/source/contributing.rst @@ -230,7 +230,14 @@ Running Google BigQuery Integration Tests You will need to create a Google BigQuery private key in JSON format in order to run Google BigQuery integration tests on your local machine and on Travis-CI. The first step is to create a `service account -`__. +`__. + +To run the integration tests locally, set the following environment variables +before running ``pytest``: + +#. ``GBQ_PROJECT_ID`` with the value being the ID of your BigQuery project. +#. ``GBQ_GOOGLE_APPLICATION_CREDENTIALS`` with the value being the *path* to + the JSON key that you downloaded for your service account. Integration tests are skipped in pull requests because the credentials that are required for running Google BigQuery integration tests are @@ -248,8 +255,8 @@ gbq integration tests on a forked repository: - ``GBQ_PROJECT_ID`` with the value being the ID of your BigQuery project. - - ``SERVICE_ACCOUNT_KEY`` with the value being the contents of the JSON key - that you downloaded for your service account. Use single quotes around + - ``SERVICE_ACCOUNT_KEY`` with the value being the *contents* of the JSON + key that you downloaded for your service account. Use single quotes around your JSON key to ensure that it is treated as a string. For both environment variables, keep the "Display value in build log" option diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py index 338d9b1eb33b..fd976ee8d3b6 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -19,9 +19,6 @@ import pandas.util.testing as tm from pandas.compat.numpy import np_datetime64_compat -PROJECT_ID = None -PRIVATE_KEY_JSON_PATH = None -PRIVATE_KEY_JSON_CONTENTS = None TABLE_ID = 'new_test' @@ -66,10 +63,7 @@ def _get_dataset_prefix_random(): def _get_project_id(): - if _in_travis_environment(): - return os.environ.get('GBQ_PROJECT_ID') - else: - return PROJECT_ID + return os.environ.get('GBQ_PROJECT_ID') def _get_private_key_path(): @@ -77,16 +71,16 @@ def _get_private_key_path(): return os.path.join(*[os.environ.get('TRAVIS_BUILD_DIR'), 'ci', 'travis_gbq.json']) else: - return PRIVATE_KEY_JSON_PATH + return os.environ.get('GBQ_GOOGLE_APPLICATION_CREDENTIALS') def _get_private_key_contents(): - if _in_travis_environment(): - with open(os.path.join(*[os.environ.get('TRAVIS_BUILD_DIR'), 'ci', - 'travis_gbq.json'])) as f: - return f.read() - else: - return PRIVATE_KEY_JSON_CONTENTS + key_path = _get_private_key_path() + if key_path is None: + return None + + with open(key_path) as f: + return f.read() def _test_imports(): From 03ccfddeee0aa4a983a190e54ab239e4829ef921 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 3 May 2017 22:17:12 -0400 Subject: [PATCH 058/519] Fixup (#34) * COMPAT: remove PandasError * migrate tests to all pytest idioms --- packages/pandas-gbq/docs/source/changelog.rst | 5 + packages/pandas-gbq/pandas_gbq/gbq.py | 23 +- .../pandas-gbq/pandas_gbq/tests/test_gbq.py | 235 ++++++++---------- 3 files changed, 123 insertions(+), 140 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 401146935017..423de73a3c86 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,6 +1,11 @@ Changelog ========= +0.1.5 / 2017-05-04 +------------------ + +- All gbq errors will simply be subclasses of ``ValueError`` and no longer inherit from the deprecated ``PandasError``. + 0.1.4 / 2017-03-17 ------------------ diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index ee42ab083097..c4c6357d1bc1 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -10,7 +10,6 @@ from distutils.version import StrictVersion from pandas import compat, DataFrame, concat -from pandas.core.common import PandasError from pandas.compat import lzip, bytes_to_str @@ -58,35 +57,35 @@ def _test_google_api_imports(): "support: {0}".format(str(e))) -class InvalidPrivateKeyFormat(PandasError, ValueError): +class InvalidPrivateKeyFormat(ValueError): """ Raised when provided private key has invalid format. """ pass -class AccessDenied(PandasError, ValueError): +class AccessDenied(ValueError): """ Raised when invalid credentials are provided, or tokens have expired. """ pass -class DatasetCreationError(PandasError, ValueError): +class DatasetCreationError(ValueError): """ Raised when the create dataset method fails """ pass -class GenericGBQException(PandasError, ValueError): +class GenericGBQException(ValueError): """ Raised when an unrecognized Google API Error occurs. """ pass -class InvalidColumnOrder(PandasError, ValueError): +class InvalidColumnOrder(ValueError): """ Raised when the provided column order for output results DataFrame does not match the schema @@ -95,7 +94,7 @@ class InvalidColumnOrder(PandasError, ValueError): pass -class InvalidIndexColumn(PandasError, ValueError): +class InvalidIndexColumn(ValueError): """ Raised when the provided index column for output results DataFrame does not match the schema @@ -104,7 +103,7 @@ class InvalidIndexColumn(PandasError, ValueError): pass -class InvalidPageToken(PandasError, ValueError): +class InvalidPageToken(ValueError): """ Raised when Google BigQuery fails to return, or returns a duplicate page token. @@ -112,7 +111,7 @@ class InvalidPageToken(PandasError, ValueError): pass -class InvalidSchema(PandasError, ValueError): +class InvalidSchema(ValueError): """ Raised when the provided DataFrame does not match the schema of the destination @@ -121,7 +120,7 @@ class InvalidSchema(PandasError, ValueError): pass -class NotFoundException(PandasError, ValueError): +class NotFoundException(ValueError): """ Raised when the project_id, table or dataset provided in the query could not be found. @@ -129,7 +128,7 @@ class NotFoundException(PandasError, ValueError): pass -class StreamingInsertError(PandasError, ValueError): +class StreamingInsertError(ValueError): """ Raised when BigQuery reports a streaming insert error. For more information see `Streaming Data Into BigQuery @@ -137,7 +136,7 @@ class StreamingInsertError(PandasError, ValueError): """ -class TableCreationError(PandasError, ValueError): +class TableCreationError(ValueError): """ Raised when the create table method fails """ diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py index fd976ee8d3b6..9386f17b905f 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -257,9 +257,9 @@ def test_generate_bq_schema_deprecated(): gbq.generate_bq_schema(df) -class TestGBQConnectorIntegrationWithLocalUserAccountAuth(tm.TestCase): +class TestGBQConnectorIntegrationWithLocalUserAccountAuth(object): - def setUp(self): + def setup_method(self, method): _setup_common() _skip_if_no_project_id() _skip_local_auth_if_in_travis_env() @@ -267,31 +267,30 @@ def setUp(self): self.sut = gbq.GbqConnector(_get_project_id()) def test_should_be_able_to_make_a_connector(self): - self.assertTrue(self.sut is not None, - 'Could not create a GbqConnector') + assert self.sut is not None, 'Could not create a GbqConnector' def test_should_be_able_to_get_valid_credentials(self): credentials = self.sut.get_credentials() - self.assertFalse(credentials.invalid, 'Returned credentials invalid') + assert credentials.invalid != 'Returned credentials invalid' def test_should_be_able_to_get_a_bigquery_service(self): bigquery_service = self.sut.get_service() - self.assertTrue(bigquery_service is not None, 'No service returned') + assert bigquery_service is not None def test_should_be_able_to_get_schema_from_query(self): schema, pages = self.sut.run_query('SELECT 1') - self.assertTrue(schema is not None) + assert schema is not None def test_should_be_able_to_get_results_from_query(self): schema, pages = self.sut.run_query('SELECT 1') - self.assertTrue(pages is not None) + assert pages is not None def test_get_application_default_credentials_does_not_throw_error(self): if _check_if_can_get_correct_default_credentials(): pytest.skip("Can get default_credentials " "from the environment!") credentials = self.sut.get_application_default_credentials() - self.assertIsNone(credentials) + assert credentials is None def test_get_application_default_credentials_returns_credentials(self): if not _check_if_can_get_correct_default_credentials(): @@ -299,11 +298,12 @@ def test_get_application_default_credentials_returns_credentials(self): "from the environment!") from oauth2client.client import GoogleCredentials credentials = self.sut.get_application_default_credentials() - self.assertTrue(isinstance(credentials, GoogleCredentials)) + assert isinstance(credentials, GoogleCredentials) -class TestGBQConnectorIntegrationWithServiceAccountKeyPath(tm.TestCase): - def setUp(self): +class TestGBQConnectorIntegrationWithServiceAccountKeyPath(object): + + def setup_method(self, method): _setup_common() _skip_if_no_project_id() @@ -313,28 +313,28 @@ def setUp(self): private_key=_get_private_key_path()) def test_should_be_able_to_make_a_connector(self): - self.assertTrue(self.sut is not None, - 'Could not create a GbqConnector') + assert self.sut is not None def test_should_be_able_to_get_valid_credentials(self): credentials = self.sut.get_credentials() - self.assertFalse(credentials.invalid, 'Returned credentials invalid') + assert not credentials.invalid def test_should_be_able_to_get_a_bigquery_service(self): bigquery_service = self.sut.get_service() - self.assertTrue(bigquery_service is not None, 'No service returned') + assert bigquery_service is not None def test_should_be_able_to_get_schema_from_query(self): schema, pages = self.sut.run_query('SELECT 1') - self.assertTrue(schema is not None) + assert schema is not None def test_should_be_able_to_get_results_from_query(self): schema, pages = self.sut.run_query('SELECT 1') - self.assertTrue(pages is not None) + assert pages is not None + +class TestGBQConnectorIntegrationWithServiceAccountKeyContents(object): -class TestGBQConnectorIntegrationWithServiceAccountKeyContents(tm.TestCase): - def setUp(self): + def setup_method(self, method): _setup_common() _skip_if_no_project_id() @@ -344,29 +344,28 @@ def setUp(self): private_key=_get_private_key_contents()) def test_should_be_able_to_make_a_connector(self): - self.assertTrue(self.sut is not None, - 'Could not create a GbqConnector') + assert self.sut is not None def test_should_be_able_to_get_valid_credentials(self): credentials = self.sut.get_credentials() - self.assertFalse(credentials.invalid, 'Returned credentials invalid') + assert not credentials.invalid def test_should_be_able_to_get_a_bigquery_service(self): bigquery_service = self.sut.get_service() - self.assertTrue(bigquery_service is not None, 'No service returned') + assert bigquery_service is not None def test_should_be_able_to_get_schema_from_query(self): schema, pages = self.sut.run_query('SELECT 1') - self.assertTrue(schema is not None) + assert schema is not None def test_should_be_able_to_get_results_from_query(self): schema, pages = self.sut.run_query('SELECT 1') - self.assertTrue(pages is not None) + assert pages is not None -class GBQUnitTests(tm.TestCase): +class GBQUnitTests(object): - def setUp(self): + def setup_method(self, method): _setup_common() def test_import_google_api_python_client(self): @@ -455,10 +454,10 @@ def test_read_gbq_with_corrupted_private_key_json_should_fail(self): private_key=re.sub('[a-z]', '9', _get_private_key_contents())) -class TestReadGBQIntegration(tm.TestCase): +class TestReadGBQIntegration(object): @classmethod - def setUpClass(cls): + def setup_class(cls): # - GLOBAL CLASS FIXTURES - # put here any instruction you want to execute only *ONCE* *BEFORE* # executing *ALL* tests described below. @@ -467,20 +466,20 @@ def setUpClass(cls): _setup_common() - def setUp(self): + def setup_method(self, method): # - PER-TEST FIXTURES - # put here any instruction you want to be run *BEFORE* *EVERY* test is # executed. pass @classmethod - def tearDownClass(cls): + def teardown_class(cls): # - GLOBAL CLASS FIXTURES - # put here any instruction you want to execute only *ONCE* *AFTER* # executing all tests. pass - def tearDown(self): + def teardown_method(self, method): # - PER-TEST FIXTURES - # put here any instructions you want to be run *AFTER* *EVERY* test is # executed. @@ -508,10 +507,10 @@ def test_should_read_as_service_account_with_key_contents(self): tm.assert_frame_equal(df, DataFrame({'valid_string': ['PI']})) -class TestReadGBQIntegrationWithServiceAccountKeyPath(tm.TestCase): +class TestReadGBQIntegrationWithServiceAccountKeyPath(object): @classmethod - def setUpClass(cls): + def setup_class(cls): # - GLOBAL CLASS FIXTURES - # put here any instruction you want to execute only *ONCE* *BEFORE* # executing *ALL* tests described below. @@ -521,7 +520,7 @@ def setUpClass(cls): _setup_common() - def setUp(self): + def setup_method(self, method): # - PER-TEST FIXTURES - # put here any instruction you want to be run *BEFORE* *EVERY* test is # executed. @@ -529,13 +528,13 @@ def setUp(self): _get_project_id(), private_key=_get_private_key_path()) @classmethod - def tearDownClass(cls): + def teardown_class(cls): # - GLOBAL CLASS FIXTURES - # put here any instruction you want to execute only *ONCE* *AFTER* # executing all tests. pass - def tearDown(self): + def teardown_method(self): # - PER-TEST FIXTURES - # put here any instructions you want to be run *AFTER* *EVERY* test is # executed. @@ -779,7 +778,7 @@ def test_download_dataset_larger_than_200k_rows(self): .format(test_size), project_id=_get_project_id(), private_key=_get_private_key_path()) - self.assertEqual(len(df.drop_duplicates()), test_size) + assert len(df.drop_duplicates()) == test_size def test_zero_rows(self): # Bug fix for https://github.com/pandas-dev/pandas/issues/10273 @@ -794,7 +793,7 @@ def test_zero_rows(self): ('is_bot', np.dtype(bool)), ('ts', 'M8[ns]')]) expected_result = DataFrame( page_array, columns=['title', 'id', 'is_bot', 'ts']) - self.assert_frame_equal(df, expected_result) + tm.assert_frame_equal(df, expected_result) def test_legacy_sql(self): legacy_sql = "SELECT id FROM [publicdata.samples.wikipedia] LIMIT 10" @@ -811,7 +810,7 @@ def test_legacy_sql(self): df = gbq.read_gbq(legacy_sql, project_id=_get_project_id(), dialect='legacy', private_key=_get_private_key_path()) - self.assertEqual(len(df.drop_duplicates()), 10) + assert len(df.drop_duplicates()) == 10 def test_standard_sql(self): standard_sql = "SELECT DISTINCT id FROM " \ @@ -828,7 +827,7 @@ def test_standard_sql(self): df = gbq.read_gbq(standard_sql, project_id=_get_project_id(), dialect='standard', private_key=_get_private_key_path()) - self.assertEqual(len(df.drop_duplicates()), 10) + assert len(df.drop_duplicates()) == 10 def test_invalid_option_for_sql_dialect(self): sql_statement = "SELECT DISTINCT id FROM " \ @@ -943,38 +942,36 @@ def test_configuration_raises_value_error_with_multiple_config(self): } } # Test that only ValueError is raised with multiple configurations - with tm.assertRaises(ValueError): + with pytest.raises(ValueError): gbq.read_gbq(sql_statement, project_id=_get_project_id(), private_key=_get_private_key_path(), configuration=config) def test_query_response_bytes(self): - self.assertEqual(self.gbq_connector.sizeof_fmt(999), "999.0 B") - self.assertEqual(self.gbq_connector.sizeof_fmt(1024), "1.0 KB") - self.assertEqual(self.gbq_connector.sizeof_fmt(1099), "1.1 KB") - self.assertEqual(self.gbq_connector.sizeof_fmt(1044480), "1020.0 KB") - self.assertEqual(self.gbq_connector.sizeof_fmt(1048576), "1.0 MB") - self.assertEqual(self.gbq_connector.sizeof_fmt(1048576000), - "1000.0 MB") - self.assertEqual(self.gbq_connector.sizeof_fmt(1073741824), "1.0 GB") - self.assertEqual(self.gbq_connector.sizeof_fmt(1.099512E12), "1.0 TB") - self.assertEqual(self.gbq_connector.sizeof_fmt(1.125900E15), "1.0 PB") - self.assertEqual(self.gbq_connector.sizeof_fmt(1.152922E18), "1.0 EB") - self.assertEqual(self.gbq_connector.sizeof_fmt(1.180592E21), "1.0 ZB") - self.assertEqual(self.gbq_connector.sizeof_fmt(1.208926E24), "1.0 YB") - self.assertEqual(self.gbq_connector.sizeof_fmt(1.208926E28), - "10000.0 YB") - - -class TestToGBQIntegrationWithServiceAccountKeyPath(tm.TestCase): + assert self.gbq_connector.sizeof_fmt(999) == "999.0 B" + assert self.gbq_connector.sizeof_fmt(1024) == "1.0 KB" + assert self.gbq_connector.sizeof_fmt(1099) == "1.1 KB" + assert self.gbq_connector.sizeof_fmt(1044480) == "1020.0 KB" + assert self.gbq_connector.sizeof_fmt(1048576) == "1.0 MB" + assert self.gbq_connector.sizeof_fmt(1048576000) == "1000.0 MB" + assert self.gbq_connector.sizeof_fmt(1073741824) == "1.0 GB" + assert self.gbq_connector.sizeof_fmt(1.099512E12) == "1.0 TB" + assert self.gbq_connector.sizeof_fmt(1.125900E15) == "1.0 PB" + assert self.gbq_connector.sizeof_fmt(1.152922E18) == "1.0 EB" + assert self.gbq_connector.sizeof_fmt(1.180592E21) == "1.0 ZB" + assert self.gbq_connector.sizeof_fmt(1.208926E24) == "1.0 YB" + assert self.gbq_connector.sizeof_fmt(1.208926E28) == "10000.0 YB" + + +class TestToGBQIntegrationWithServiceAccountKeyPath(object): # Changes to BigQuery table schema may take up to 2 minutes as of May 2015 # As a workaround to this issue, each test should use a unique table name. - # Make sure to modify the for loop range in the tearDownClass when a new + # Make sure to modify the for loop range in the teardown_class when a new # test is added See `Issue 191 # `__ @classmethod - def setUpClass(cls): + def setup_class(cls): # - GLOBAL CLASS FIXTURES - # put here any instruction you want to execute only *ONCE* *BEFORE* # executing *ALL* tests described below. @@ -984,7 +981,7 @@ def setUpClass(cls): _setup_common() - def setUp(self): + def setup_method(self, method): # - PER-TEST FIXTURES - # put here any instruction you want to be run *BEFORE* *EVERY* test is # executed. @@ -1002,13 +999,13 @@ def setUp(self): self.dataset.create(self.dataset_prefix + "1") @classmethod - def tearDownClass(cls): + def teardown_class(cls): # - GLOBAL CLASS FIXTURES - # put here any instruction you want to execute only *ONCE* *AFTER* # executing all tests. pass - def tearDown(self): + def teardown_method(self, method): # - PER-TEST FIXTURES - # put here any instructions you want to be run *AFTER* *EVERY* test is # executed. @@ -1028,7 +1025,7 @@ def test_upload_data(self): .format(self.destination_table + test_id), project_id=_get_project_id(), private_key=_get_private_key_path()) - self.assertEqual(result['num_rows'][0], test_size) + assert result['num_rows'][0] == test_size def test_upload_data_if_table_exists_fail(self): test_id = "2" @@ -1066,7 +1063,7 @@ def test_upload_data_if_table_exists_append(self): .format(self.destination_table + test_id), project_id=_get_project_id(), private_key=_get_private_key_path()) - self.assertEqual(result['num_rows'][0], test_size * 2) + assert result['num_rows'][0] == test_size * 2 # Try inserting with a different schema, confirm failure with tm.assertRaises(gbq.InvalidSchema): @@ -1095,7 +1092,7 @@ def test_upload_data_if_table_exists_replace(self): .format(self.destination_table + test_id), project_id=_get_project_id(), private_key=_get_private_key_path()) - self.assertEqual(result['num_rows'][0], 5) + assert result['num_rows'][0] == 5 def test_upload_data_if_table_exists_raises_value_error(self): test_id = "4" @@ -1130,26 +1127,24 @@ def test_generate_schema(self): {'name': 'C', 'type': 'STRING'}, {'name': 'D', 'type': 'TIMESTAMP'}]} - self.assertEqual(schema, test_schema) + assert schema == test_schema def test_create_table(self): test_id = "6" schema = gbq._generate_bq_schema(tm.makeMixedDataFrame()) self.table.create(TABLE_ID + test_id, schema) - self.assertTrue(self.table.exists(TABLE_ID + test_id), - 'Expected table to exist') + assert self.table.exists(TABLE_ID + test_id) def test_create_table_already_exists(self): test_id = "6" schema = gbq._generate_bq_schema(tm.makeMixedDataFrame()) self.table.create(TABLE_ID + test_id, schema) - with tm.assertRaises(gbq.TableCreationError): + with pytest.raises(gbq.TableCreationError): self.table.create(TABLE_ID + test_id, schema) def test_table_does_not_exist(self): test_id = "7" - self.assertTrue(not self.table.exists(TABLE_ID + test_id), - 'Expected table not to exist') + assert not self.table.exists(TABLE_ID + test_id) def test_delete_table(self): test_id = "8" @@ -1159,11 +1154,11 @@ def test_delete_table(self): {'name': 'D', 'type': 'TIMESTAMP'}]} self.table.create(TABLE_ID + test_id, test_schema) self.table.delete(TABLE_ID + test_id) - self.assertTrue(not self.table.exists( - TABLE_ID + test_id), 'Expected table not to exist') + assert not self.table.exists( + TABLE_ID + test_id) def test_delete_table_not_found(self): - with tm.assertRaises(gbq.NotFoundException): + with pytest.raises(gbq.NotFoundException): self.table.delete(TABLE_ID + "not_found") def test_list_table(self): @@ -1173,10 +1168,8 @@ def test_list_table(self): {'name': 'C', 'type': 'STRING'}, {'name': 'D', 'type': 'TIMESTAMP'}]} self.table.create(TABLE_ID + test_id, test_schema) - self.assertTrue(TABLE_ID + test_id in - self.dataset.tables(self.dataset_prefix + "1"), - 'Expected table list to contain table {0}' - .format(TABLE_ID + test_id)) + assert TABLE_ID + test_id in self.dataset.tables( + self.dataset_prefix + "1") def test_verify_schema_allows_flexible_column_order(self): test_id = "10" @@ -1190,9 +1183,8 @@ def test_verify_schema_allows_flexible_column_order(self): {'name': 'D', 'type': 'TIMESTAMP'}]} self.table.create(TABLE_ID + test_id, test_schema_1) - self.assertTrue(self.sut.verify_schema( - self.dataset_prefix + "1", TABLE_ID + test_id, test_schema_2), - 'Expected schema to match') + assert self.sut.verify_schema( + self.dataset_prefix + "1", TABLE_ID + test_id, test_schema_2) def test_verify_schema_fails_different_data_type(self): test_id = "11" @@ -1206,9 +1198,8 @@ def test_verify_schema_fails_different_data_type(self): {'name': 'D', 'type': 'TIMESTAMP'}]} self.table.create(TABLE_ID + test_id, test_schema_1) - self.assertFalse(self.sut.verify_schema( - self.dataset_prefix + "1", TABLE_ID + test_id, test_schema_2), - 'Expected different schema') + assert not self.sut.verify_schema(self.dataset_prefix + "1", + TABLE_ID + test_id, test_schema_2) def test_verify_schema_fails_different_structure(self): test_id = "12" @@ -1222,9 +1213,8 @@ def test_verify_schema_fails_different_structure(self): {'name': 'D', 'type': 'TIMESTAMP'}]} self.table.create(TABLE_ID + test_id, test_schema_1) - self.assertFalse(self.sut.verify_schema( - self.dataset_prefix + "1", TABLE_ID + test_id, test_schema_2), - 'Expected different schema') + assert not self.sut.verify_schema( + self.dataset_prefix + "1", TABLE_ID + test_id, test_schema_2) def test_upload_data_flexible_column_order(self): test_id = "13" @@ -1265,15 +1255,12 @@ def test_verify_schema_ignores_field_mode(self): 'type': 'TIMESTAMP'}]} self.table.create(TABLE_ID + test_id, test_schema_1) - self.assertTrue(self.sut.verify_schema( - self.dataset_prefix + "1", TABLE_ID + test_id, test_schema_2), - 'Expected schema to match') + assert self.sut.verify_schema( + self.dataset_prefix + "1", TABLE_ID + test_id, test_schema_2) def test_list_dataset(self): dataset_id = self.dataset_prefix + "1" - self.assertTrue(dataset_id in self.dataset.datasets(), - 'Expected dataset list to contain dataset {0}' - .format(dataset_id)) + assert dataset_id in self.dataset.datasets() def test_list_table_zero_results(self): dataset_id = self.dataset_prefix + "2" @@ -1281,38 +1268,34 @@ def test_list_table_zero_results(self): table_list = gbq._Dataset(_get_project_id(), private_key=_get_private_key_path() ).tables(dataset_id) - self.assertEqual(len(table_list), 0, - 'Expected gbq.list_table() to return 0') + assert len(table_list) == 0 def test_create_dataset(self): dataset_id = self.dataset_prefix + "3" self.dataset.create(dataset_id) - self.assertTrue(dataset_id in self.dataset.datasets(), - 'Expected dataset to exist') + assert dataset_id in self.dataset.datasets() def test_create_dataset_already_exists(self): dataset_id = self.dataset_prefix + "3" self.dataset.create(dataset_id) - with tm.assertRaises(gbq.DatasetCreationError): + with pytest.raises(gbq.DatasetCreationError): self.dataset.create(dataset_id) def test_delete_dataset(self): dataset_id = self.dataset_prefix + "4" self.dataset.create(dataset_id) self.dataset.delete(dataset_id) - self.assertTrue(dataset_id not in self.dataset.datasets(), - 'Expected dataset not to exist') + assert dataset_id not in self.dataset.datasets() def test_delete_dataset_not_found(self): dataset_id = self.dataset_prefix + "not_found" - with tm.assertRaises(gbq.NotFoundException): + with pytest.raises(gbq.NotFoundException): self.dataset.delete(dataset_id) def test_dataset_exists(self): dataset_id = self.dataset_prefix + "5" self.dataset.create(dataset_id) - self.assertTrue(self.dataset.exists(dataset_id), - 'Expected dataset to exist') + assert self.dataset.exists(dataset_id) def create_table_data_dataset_does_not_exist(self): dataset_id = self.dataset_prefix + "6" @@ -1320,27 +1303,23 @@ def create_table_data_dataset_does_not_exist(self): table_with_new_dataset = gbq._Table(_get_project_id(), dataset_id) df = make_mixed_dataframe_v2(10) table_with_new_dataset.create(table_id, gbq._generate_bq_schema(df)) - self.assertTrue(self.dataset.exists(dataset_id), - 'Expected dataset to exist') - self.assertTrue(table_with_new_dataset.exists( - table_id), 'Expected dataset to exist') + assert self.dataset.exists(dataset_id) + assert table_with_new_dataset.exists(table_id) def test_dataset_does_not_exist(self): - self.assertTrue(not self.dataset.exists( - self.dataset_prefix + "_not_found"), - 'Expected dataset not to exist') + assert not self.dataset.exists(self.dataset_prefix + "_not_found") -class TestToGBQIntegrationWithLocalUserAccountAuth(tm.TestCase): +class TestToGBQIntegrationWithLocalUserAccountAuth(object): # Changes to BigQuery table schema may take up to 2 minutes as of May 2015 # As a workaround to this issue, each test should use a unique table name. - # Make sure to modify the for loop range in the tearDownClass when a new + # Make sure to modify the for loop range in the teardown_class when a new # test is added # See `Issue 191 # `__ @classmethod - def setUpClass(cls): + def setup_class(cls): # - GLOBAL CLASS FIXTURES - # put here any instruction you want to execute only *ONCE* *BEFORE* # executing *ALL* tests described below. @@ -1350,7 +1329,7 @@ def setUpClass(cls): _setup_common() - def setUp(self): + def setup_method(self, method): # - PER-TEST FIXTURES - # put here any instruction you want to be run *BEFORE* *EVERY* test # is executed. @@ -1361,13 +1340,13 @@ def setUp(self): TABLE_ID) @classmethod - def tearDownClass(cls): + def teardown_class(cls): # - GLOBAL CLASS FIXTURES - # put here any instruction you want to execute only *ONCE* *AFTER* # executing all tests. pass - def tearDown(self): + def teardown_method(self): # - PER-TEST FIXTURES - # put here any instructions you want to be run *AFTER* *EVERY* test # is executed. @@ -1387,19 +1366,19 @@ def test_upload_data(self): self.destination_table + test_id), project_id=_get_project_id()) - self.assertEqual(result['num_rows'][0], test_size) + assert result['num_rows'][0] == test_size -class TestToGBQIntegrationWithServiceAccountKeyContents(tm.TestCase): +class TestToGBQIntegrationWithServiceAccountKeyContents(object): # Changes to BigQuery table schema may take up to 2 minutes as of May 2015 # As a workaround to this issue, each test should use a unique table name. - # Make sure to modify the for loop range in the tearDownClass when a new + # Make sure to modify the for loop range in the teardown_class when a new # test is added # See `Issue 191 # `__ @classmethod - def setUpClass(cls): + def setup_class(cls): # - GLOBAL CLASS FIXTURES - # put here any instruction you want to execute only *ONCE* *BEFORE* # executing *ALL* tests described below. @@ -1409,7 +1388,7 @@ def setUpClass(cls): _skip_if_no_private_key_contents() - def setUp(self): + def setup_method(self, method): # - PER-TEST FIXTURES - # put here any instruction you want to be run *BEFORE* *EVERY* test # is executed. @@ -1419,13 +1398,13 @@ def setUp(self): TABLE_ID) @classmethod - def tearDownClass(cls): + def teardown_class(cls): # - GLOBAL CLASS FIXTURES - # put here any instruction you want to execute only *ONCE* *AFTER* # executing all tests. pass - def tearDown(self): + def teardown_method(self, method): # - PER-TEST FIXTURES - # put here any instructions you want to be run *AFTER* *EVERY* test # is executed. @@ -1445,4 +1424,4 @@ def test_upload_data(self): self.destination_table + test_id), project_id=_get_project_id(), private_key=_get_private_key_contents()) - self.assertEqual(result['num_rows'][0], test_size) + assert result['num_rows'][0] == test_size From aca918547b882718425ebb053183d0377fbb5035 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 3 May 2017 22:33:01 -0400 Subject: [PATCH 059/519] RLS 0.1.6 --- packages/pandas-gbq/docs/source/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 423de73a3c86..044485a0dff0 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,7 +1,7 @@ Changelog ========= -0.1.5 / 2017-05-04 +0.1.6 / 2017-05-03 ------------------ - All gbq errors will simply be subclasses of ``ValueError`` and no longer inherit from the deprecated ``PandasError``. From 8d01a6943f2888447c1065e4e5ea0de7a7eab64b Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Fri, 12 May 2017 06:35:13 -0400 Subject: [PATCH 060/519] next release in changelog.rst --- packages/pandas-gbq/docs/source/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 044485a0dff0..ee364edd60e4 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,6 +1,9 @@ Changelog ========= +0.1.7 / 2017-??-?? +------------------ + 0.1.6 / 2017-05-03 ------------------ From 9bbbb9431e58262d4c8202886cbbf208847695e0 Mon Sep 17 00:00:00 2001 From: Luca Fiaschi Date: Sat, 13 May 2017 17:37:25 +0200 Subject: [PATCH 061/519] BUG: Propagate command line arguments (#35) --- packages/pandas-gbq/docs/source/changelog.rst | 2 ++ packages/pandas-gbq/pandas_gbq/gbq.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index ee364edd60e4..677cfcc586c6 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -4,6 +4,8 @@ Changelog 0.1.7 / 2017-??-?? ------------------ +- Resolve issue where the optional ``--noauth_local_webserver`` command line argument would not be propagated during the authentication process. + 0.1.6 / 2017-05-03 ------------------ diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index c4c6357d1bc1..e584bd8f2975 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -231,7 +231,7 @@ def get_user_account_credentials(self): credentials = storage.get() if credentials is None or credentials.invalid or self.reauth: - credentials = run_flow(flow, storage, argparser.parse_args([])) + credentials = run_flow(flow, storage, argparser.parse_args()) return credentials From 4a75b117854e0a97fd7373e44e9a90faa67942db Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Thu, 18 May 2017 06:17:48 -0400 Subject: [PATCH 062/519] COMPAT: drop 3.4 support (#40) TST: drop 3.4, add 3.6/0.20.1 to config matrix PEP setup.py --- packages/pandas-gbq/.travis.yml | 16 +++++++++++----- ...ments-2.7.pip => requirements-2.7-0.19.2.pip} | 0 ...ments-3.4.pip => requirements-3.5-0.18.1.pip} | 0 ...ments-3.5.pip => requirements-3.6-0.20.1.pip} | 0 ...ments-3.6.pip => requirements-3.6-MASTER.pip} | 0 packages/pandas-gbq/docs/source/changelog.rst | 9 +++++---- packages/pandas-gbq/setup.py | 8 ++++---- 7 files changed, 20 insertions(+), 13 deletions(-) rename packages/pandas-gbq/ci/{requirements-2.7.pip => requirements-2.7-0.19.2.pip} (100%) rename packages/pandas-gbq/ci/{requirements-3.4.pip => requirements-3.5-0.18.1.pip} (100%) rename packages/pandas-gbq/ci/{requirements-3.5.pip => requirements-3.6-0.20.1.pip} (100%) rename packages/pandas-gbq/ci/{requirements-3.6.pip => requirements-3.6-MASTER.pip} (100%) diff --git a/packages/pandas-gbq/.travis.yml b/packages/pandas-gbq/.travis.yml index 5a858c3f2297..91b41be4cef7 100644 --- a/packages/pandas-gbq/.travis.yml +++ b/packages/pandas-gbq/.travis.yml @@ -4,9 +4,9 @@ language: python env: - PYTHON=2.7 PANDAS=0.19.2 COVERAGE='false' LINT='false' - - PYTHON=3.4 PANDAS=0.18.1 COVERAGE='false' LINT='false' - - PYTHON=3.5 PANDAS=0.19.2 COVERAGE='true' LINT='false' - - PYTHON=3.6 PANDAS=0.19.2 COVERAGE='false' LINT='true' + - PYTHON=3.5 PANDAS=0.18.1 COVERAGE='true' LINT='false' + - PYTHON=3.6 PANDAS=0.20.1 COVERAGE='false' LINT='true' + - PYTHON=3.6 PANDAS=MASTER COVERAGE='false' LINT='false' before_install: - echo "before_install" @@ -23,9 +23,15 @@ install: - conda info -a - conda create -n test-environment python=$PYTHON - source activate test-environment - - conda install pandas=$PANDAS + - if [[ "$PANDAS" == "MASTER" ]]; then + conda install numpy pytz python-dateutil; + PRE_WHEELS="https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com"; + pip install --pre --upgrade --timeout=60 -f $PRE_WHEELS pandas; + else + conda install pandas=$PANDAS; + fi - pip install coverage pytest pytest-cov flake8 codecov - - REQ="ci/requirements-${PYTHON}.pip" + - REQ="ci/requirements-${PYTHON}-${PANDAS}.pip" - pip install -r $REQ - conda list - python setup.py install diff --git a/packages/pandas-gbq/ci/requirements-2.7.pip b/packages/pandas-gbq/ci/requirements-2.7-0.19.2.pip similarity index 100% rename from packages/pandas-gbq/ci/requirements-2.7.pip rename to packages/pandas-gbq/ci/requirements-2.7-0.19.2.pip diff --git a/packages/pandas-gbq/ci/requirements-3.4.pip b/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip similarity index 100% rename from packages/pandas-gbq/ci/requirements-3.4.pip rename to packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip diff --git a/packages/pandas-gbq/ci/requirements-3.5.pip b/packages/pandas-gbq/ci/requirements-3.6-0.20.1.pip similarity index 100% rename from packages/pandas-gbq/ci/requirements-3.5.pip rename to packages/pandas-gbq/ci/requirements-3.6-0.20.1.pip diff --git a/packages/pandas-gbq/ci/requirements-3.6.pip b/packages/pandas-gbq/ci/requirements-3.6-MASTER.pip similarity index 100% rename from packages/pandas-gbq/ci/requirements-3.6.pip rename to packages/pandas-gbq/ci/requirements-3.6-MASTER.pip diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 677cfcc586c6..53c1a1b94d1a 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,10 +1,11 @@ Changelog ========= -0.1.7 / 2017-??-?? +0.2.0 / 2017-??-?? ------------------ -- Resolve issue where the optional ``--noauth_local_webserver`` command line argument would not be propagated during the authentication process. +- Resolve issue where the optional ``--noauth_local_webserver`` command line argument would not be propagated during the authentication process. (:issue:`35`) +- Drop support for Python 3.4 (:issue:`40`) 0.1.6 / 2017-05-03 ------------------ @@ -14,12 +15,12 @@ Changelog 0.1.4 / 2017-03-17 ------------------ -- ``InvalidIndexColumn`` will be raised instead of ``InvalidColumnOrder`` in ``read_gbq`` when the index column specified does not exist in the BigQuery schema. :issue:`6` +- ``InvalidIndexColumn`` will be raised instead of ``InvalidColumnOrder`` in ``read_gbq`` when the index column specified does not exist in the BigQuery schema. (:issue:`6`) 0.1.3 / 2017-03-04 ------------------ -- Bug with appending to a BigQuery table where fields have modes (NULLABLE,REQUIRED,REPEATED) specified. These modes were compared versus the remote schema and writing a table via ``to_gbq`` would previously raise. :issue:`13` +- Bug with appending to a BigQuery table where fields have modes (NULLABLE,REQUIRED,REPEATED) specified. These modes were compared versus the remote schema and writing a table via ``to_gbq`` would previously raise. (:issue:`13`) 0.1.2 / 2017-02-23 ------------------ diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index 3aa08a64b29f..a3b8f06f7ed8 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -1,26 +1,27 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from ast import parse -import os from setuptools import setup, find_packages +import versioneer NAME = 'pandas-gbq' # versioning -import versioneer cmdclass = versioneer.get_cmdclass() + def readme(): with open('README.rst') as f: return f.read() + INSTALL_REQUIRES = ( ['pandas', 'httplib2', 'google-api-python-client', 'oauth2client'] ) + setup( name=NAME, version=versioneer.get_version(), @@ -40,7 +41,6 @@ def readme(): 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Topic :: Scientific/Engineering', From a3f2290e96a1c262e0f71397ac3931e3b7365d22 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 9 Jun 2017 15:26:54 -0700 Subject: [PATCH 063/519] CLN: make license file machine readable (#56) Splits extra information about the license and copyright holders to AUTHORS.md. Uses plain text license so that string matching of the license file works better. Fixes GH#30. --- packages/pandas-gbq/AUTHORS.md | 57 +++++++++++++++++++++ packages/pandas-gbq/LICENSE.md | 87 --------------------------------- packages/pandas-gbq/LICENSE.txt | 29 +++++++++++ 3 files changed, 86 insertions(+), 87 deletions(-) create mode 100644 packages/pandas-gbq/AUTHORS.md delete mode 100644 packages/pandas-gbq/LICENSE.md create mode 100644 packages/pandas-gbq/LICENSE.txt diff --git a/packages/pandas-gbq/AUTHORS.md b/packages/pandas-gbq/AUTHORS.md new file mode 100644 index 000000000000..dcaaea101f4c --- /dev/null +++ b/packages/pandas-gbq/AUTHORS.md @@ -0,0 +1,57 @@ +About the Copyright Holders +=========================== + +* Copyright (c) 2008-2011 AQR Capital Management, LLC + + AQR Capital Management began pandas development in 2008. Development was + led by Wes McKinney. AQR released the source under this license in 2009. +* Copyright (c) 2011-2012, Lambda Foundry, Inc. + + Wes is now an employee of Lambda Foundry, and remains the pandas project + lead. +* Copyright (c) 2011-2012, PyData Development Team + + The PyData Development Team is the collection of developers of the PyData + project. This includes all of the PyData sub-projects, including pandas. The + core team that coordinates development on GitHub can be found here: + http://github.com/pydata. + +Full credits for pandas contributors can be found in the documentation. + +Our Copyright Policy +==================== + +PyData uses a shared copyright model. Each contributor maintains copyright +over their contributions to PyData. However, it is important to note that +these contributions are typically only changes to the repositories. Thus, +the PyData source code, in its entirety, is not the copyright of any single +person or institution. Instead, it is the collective copyright of the +entire PyData Development Team. If individual contributors want to maintain +a record of what changes/contributions they have specific copyright on, +they should indicate their copyright in the commit message of the change +when they commit the change to one of the PyData repositories. + +With this in mind, the following banner should be used in any source code +file to indicate the copyright and license terms: + +``` +#----------------------------------------------------------------------------- +# Copyright (c) 2012, PyData Development Team +# All rights reserved. +# +# Distributed under the terms of the BSD Simplified License. +# +# The full license is in the LICENSE file, distributed with this software. +#----------------------------------------------------------------------------- +``` + +Other licenses can be found in the LICENSES directory. + +License +======= + +pandas is distributed under a 3-clause ("Simplified" or "New") BSD +license. Parts of NumPy, SciPy, numpydoc, bottleneck, which all have +BSD-compatible licenses, are included. Their licenses follow the pandas +license. + diff --git a/packages/pandas-gbq/LICENSE.md b/packages/pandas-gbq/LICENSE.md deleted file mode 100644 index 474cd65bfb48..000000000000 --- a/packages/pandas-gbq/LICENSE.md +++ /dev/null @@ -1,87 +0,0 @@ -======= -License -======= - -pandas is distributed under a 3-clause ("Simplified" or "New") BSD -license. Parts of NumPy, SciPy, numpydoc, bottleneck, which all have -BSD-compatible licenses, are included. Their licenses follow the pandas -license. - -pandas license -============== - -Copyright (c) 2011-2012, Lambda Foundry, Inc. and PyData Development Team -All rights reserved. - -Copyright (c) 2008-2011 AQR Capital Management, LLC -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - - * Neither the name of the copyright holder nor the names of any - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -About the Copyright Holders -=========================== - -AQR Capital Management began pandas development in 2008. Development was -led by Wes McKinney. AQR released the source under this license in 2009. -Wes is now an employee of Lambda Foundry, and remains the pandas project -lead. - -The PyData Development Team is the collection of developers of the PyData -project. This includes all of the PyData sub-projects, including pandas. The -core team that coordinates development on GitHub can be found here: -http://github.com/pydata. - -Full credits for pandas contributors can be found in the documentation. - -Our Copyright Policy -==================== - -PyData uses a shared copyright model. Each contributor maintains copyright -over their contributions to PyData. However, it is important to note that -these contributions are typically only changes to the repositories. Thus, -the PyData source code, in its entirety, is not the copyright of any single -person or institution. Instead, it is the collective copyright of the -entire PyData Development Team. If individual contributors want to maintain -a record of what changes/contributions they have specific copyright on, -they should indicate their copyright in the commit message of the change -when they commit the change to one of the PyData repositories. - -With this in mind, the following banner should be used in any source code -file to indicate the copyright and license terms: - -#----------------------------------------------------------------------------- -# Copyright (c) 2012, PyData Development Team -# All rights reserved. -# -# Distributed under the terms of the BSD Simplified License. -# -# The full license is in the LICENSE file, distributed with this software. -#----------------------------------------------------------------------------- - -Other licenses can be found in the LICENSES directory. diff --git a/packages/pandas-gbq/LICENSE.txt b/packages/pandas-gbq/LICENSE.txt new file mode 100644 index 000000000000..924de26253bf --- /dev/null +++ b/packages/pandas-gbq/LICENSE.txt @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2008-2012, AQR Capital Management, LLC, Lambda Foundry, Inc. and PyData Development Team +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. From 55e1f8e9a41a27934b057907b3dac12b5707f8f6 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 9 Jun 2017 18:27:40 -0400 Subject: [PATCH 064/519] TST: Run flake8 in Travis-CI python 2.7 build (#54) --- packages/pandas-gbq/.travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/.travis.yml b/packages/pandas-gbq/.travis.yml index 91b41be4cef7..881974df4998 100644 --- a/packages/pandas-gbq/.travis.yml +++ b/packages/pandas-gbq/.travis.yml @@ -3,7 +3,7 @@ sudo: false language: python env: - - PYTHON=2.7 PANDAS=0.19.2 COVERAGE='false' LINT='false' + - PYTHON=2.7 PANDAS=0.19.2 COVERAGE='false' LINT='true' - PYTHON=3.5 PANDAS=0.18.1 COVERAGE='true' LINT='false' - PYTHON=3.6 PANDAS=0.20.1 COVERAGE='false' LINT='true' - PYTHON=3.6 PANDAS=MASTER COVERAGE='false' LINT='false' From d09cae276edf65a9f7177560d59688cff2079b55 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Sat, 10 Jun 2017 23:11:12 -0400 Subject: [PATCH 065/519] TST: Fix failing integration tests (#58) * TST: Fix broken tests failing with 'NoneType' object is not iterable * MAINT: pandas.util.testing.assertRaises removed This method was removed in https://github.com/pandas-dev/pandas/pull/16089 in favor of pytest.raises. * MAINT: pandas.util.testing.assert_equals removed This method was removed in https://github.com/pandas-dev/pandas/pull/16017 in favor of pytest.raises. --- packages/pandas-gbq/pandas_gbq/gbq.py | 3 + .../pandas-gbq/pandas_gbq/tests/test_gbq.py | 62 +++++++++---------- 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index e584bd8f2975..8d3891003121 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -1055,6 +1055,9 @@ def datasets(self): dataset_response = list_dataset_response.get('datasets') next_page_token = list_dataset_response.get('nextPageToken') + if dataset_response is None: + dataset_response = [] + for row_num, raw_row in enumerate(dataset_response): dataset_list.append( raw_row['datasetReference']['datasetId']) diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py index 9386f17b905f..2ca57b1686a1 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -375,7 +375,7 @@ def test_import_google_api_python_client(self): "using google-api-python-client==1.2") if compat.PY2: - with tm.assertRaises(ImportError): + with pytest.raises(ImportError): from googleapiclient.discovery import build # noqa from googleapiclient.errors import HttpError # noqa from apiclient.discovery import build # noqa @@ -386,34 +386,34 @@ def test_import_google_api_python_client(self): def test_should_return_bigquery_integers_as_python_ints(self): result = gbq._parse_entry(1, 'INTEGER') - tm.assert_equal(result, int(1)) + assert result == int(1) def test_should_return_bigquery_floats_as_python_floats(self): result = gbq._parse_entry(1, 'FLOAT') - tm.assert_equal(result, float(1)) + assert result == float(1) def test_should_return_bigquery_timestamps_as_numpy_datetime(self): result = gbq._parse_entry('0e9', 'TIMESTAMP') - tm.assert_equal(result, np_datetime64_compat('1970-01-01T00:00:00Z')) + assert result == np_datetime64_compat('1970-01-01T00:00:00Z') def test_should_return_bigquery_booleans_as_python_booleans(self): result = gbq._parse_entry('false', 'BOOLEAN') - tm.assert_equal(result, False) + assert not result def test_should_return_bigquery_strings_as_python_strings(self): result = gbq._parse_entry('STRING', 'STRING') - tm.assert_equal(result, 'STRING') + assert result == 'STRING' def test_to_gbq_should_fail_if_invalid_table_name_passed(self): - with tm.assertRaises(gbq.NotFoundException): + with pytest.raises(gbq.NotFoundException): gbq.to_gbq(DataFrame(), 'invalid_table_name', project_id="1234") def test_to_gbq_with_no_project_id_given_should_fail(self): - with tm.assertRaises(TypeError): + with pytest.raises(TypeError): gbq.to_gbq(DataFrame(), 'dataset.tablename') def test_read_gbq_with_no_project_id_given_should_fail(self): - with tm.assertRaises(TypeError): + with pytest.raises(TypeError): gbq.read_gbq('SELECT 1') def test_that_parse_data_works_properly(self): @@ -426,29 +426,29 @@ def test_that_parse_data_works_properly(self): tm.assert_frame_equal(test_output, correct_output) def test_read_gbq_with_invalid_private_key_json_should_fail(self): - with tm.assertRaises(gbq.InvalidPrivateKeyFormat): + with pytest.raises(gbq.InvalidPrivateKeyFormat): gbq.read_gbq('SELECT 1', project_id='x', private_key='y') def test_read_gbq_with_empty_private_key_json_should_fail(self): - with tm.assertRaises(gbq.InvalidPrivateKeyFormat): + with pytest.raises(gbq.InvalidPrivateKeyFormat): gbq.read_gbq('SELECT 1', project_id='x', private_key='{}') def test_read_gbq_with_private_key_json_wrong_types_should_fail(self): - with tm.assertRaises(gbq.InvalidPrivateKeyFormat): + with pytest.raises(gbq.InvalidPrivateKeyFormat): gbq.read_gbq( 'SELECT 1', project_id='x', private_key='{ "client_email" : 1, "private_key" : True }') def test_read_gbq_with_empty_private_key_file_should_fail(self): with tm.ensure_clean() as empty_file_path: - with tm.assertRaises(gbq.InvalidPrivateKeyFormat): + with pytest.raises(gbq.InvalidPrivateKeyFormat): gbq.read_gbq('SELECT 1', project_id='x', private_key=empty_file_path) def test_read_gbq_with_corrupted_private_key_json_should_fail(self): _skip_if_no_private_key_contents() - with tm.assertRaises(gbq.InvalidPrivateKeyFormat): + with pytest.raises(gbq.InvalidPrivateKeyFormat): gbq.read_gbq( 'SELECT 1', project_id='x', private_key=re.sub('[a-z]', '9', _get_private_key_contents())) @@ -708,7 +708,7 @@ def test_index_column(self): private_key=_get_private_key_path()) correct_frame = DataFrame( {'string_1': ['a'], 'string_2': ['b']}).set_index("string_1") - tm.assert_equal(result_frame.index.name, correct_frame.index.name) + assert result_frame.index.name == correct_frame.index.name def test_column_order(self): query = "SELECT 'a' AS string_1, 'b' AS string_2, 'c' AS string_3" @@ -725,7 +725,7 @@ def test_read_gbq_raises_invalid_column_order(self): col_order = ['string_aaa', 'string_1', 'string_2'] # Column string_aaa does not exist. Should raise InvalidColumnOrder - with tm.assertRaises(gbq.InvalidColumnOrder): + with pytest.raises(gbq.InvalidColumnOrder): gbq.read_gbq(query, project_id=_get_project_id(), col_order=col_order, private_key=_get_private_key_path()) @@ -747,24 +747,24 @@ def test_read_gbq_raises_invalid_index_column(self): col_order = ['string_3', 'string_2'] # Column string_bbb does not exist. Should raise InvalidIndexColumn - with tm.assertRaises(gbq.InvalidIndexColumn): + with pytest.raises(gbq.InvalidIndexColumn): gbq.read_gbq(query, project_id=_get_project_id(), index_col='string_bbb', col_order=col_order, private_key=_get_private_key_path()) def test_malformed_query(self): - with tm.assertRaises(gbq.GenericGBQException): + with pytest.raises(gbq.GenericGBQException): gbq.read_gbq("SELCET * FORM [publicdata:samples.shakespeare]", project_id=_get_project_id(), private_key=_get_private_key_path()) def test_bad_project_id(self): - with tm.assertRaises(gbq.GenericGBQException): + with pytest.raises(gbq.GenericGBQException): gbq.read_gbq("SELECT 1", project_id='001', private_key=_get_private_key_path()) def test_bad_table_name(self): - with tm.assertRaises(gbq.GenericGBQException): + with pytest.raises(gbq.GenericGBQException): gbq.read_gbq("SELECT * FROM [publicdata:samples.nope]", project_id=_get_project_id(), private_key=_get_private_key_path()) @@ -800,7 +800,7 @@ def test_legacy_sql(self): # Test that a legacy sql statement fails when # setting dialect='standard' - with tm.assertRaises(gbq.GenericGBQException): + with pytest.raises(gbq.GenericGBQException): gbq.read_gbq(legacy_sql, project_id=_get_project_id(), dialect='standard', private_key=_get_private_key_path()) @@ -818,7 +818,7 @@ def test_standard_sql(self): # Test that a standard sql statement fails when using # the legacy SQL dialect (default value) - with tm.assertRaises(gbq.GenericGBQException): + with pytest.raises(gbq.GenericGBQException): gbq.read_gbq(standard_sql, project_id=_get_project_id(), private_key=_get_private_key_path()) @@ -834,7 +834,7 @@ def test_invalid_option_for_sql_dialect(self): "`publicdata.samples.wikipedia` LIMIT 10" # Test that an invalid option for `dialect` raises ValueError - with tm.assertRaises(ValueError): + with pytest.raises(ValueError): gbq.read_gbq(sql_statement, project_id=_get_project_id(), dialect='invalid', private_key=_get_private_key_path()) @@ -874,7 +874,7 @@ def test_query_with_parameters(self): } # Test that a query that relies on parameters fails # when parameters are not supplied via configuration - with tm.assertRaises(ValueError): + with pytest.raises(ValueError): gbq.read_gbq(sql_statement, project_id=_get_project_id(), private_key=_get_private_key_path()) @@ -896,7 +896,7 @@ def test_query_inside_configuration(self): } # Test that it can't pass query both # inside config and as parameter - with tm.assertRaises(ValueError): + with pytest.raises(ValueError): gbq.read_gbq(query_no_use, project_id=_get_project_id(), private_key=_get_private_key_path(), configuration=config) @@ -924,7 +924,7 @@ def test_configuration_without_query(self): } # Test that only 'query' configurations are supported # nor 'copy','load','extract' - with tm.assertRaises(ValueError): + with pytest.raises(ValueError): gbq.read_gbq(sql_statement, project_id=_get_project_id(), private_key=_get_private_key_path(), configuration=config) @@ -1034,12 +1034,12 @@ def test_upload_data_if_table_exists_fail(self): self.table.create(TABLE_ID + test_id, gbq._generate_bq_schema(df)) # Test the default value of if_exists is 'fail' - with tm.assertRaises(gbq.TableCreationError): + with pytest.raises(gbq.TableCreationError): gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), private_key=_get_private_key_path()) # Test the if_exists parameter with value 'fail' - with tm.assertRaises(gbq.TableCreationError): + with pytest.raises(gbq.TableCreationError): gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), if_exists='fail', private_key=_get_private_key_path()) @@ -1066,7 +1066,7 @@ def test_upload_data_if_table_exists_append(self): assert result['num_rows'][0] == test_size * 2 # Try inserting with a different schema, confirm failure - with tm.assertRaises(gbq.InvalidSchema): + with pytest.raises(gbq.InvalidSchema): gbq.to_gbq(df_different_schema, self.destination_table + test_id, _get_project_id(), if_exists='append', private_key=_get_private_key_path()) @@ -1100,7 +1100,7 @@ def test_upload_data_if_table_exists_raises_value_error(self): df = make_mixed_dataframe_v2(test_size) # Test invalid value for if_exists parameter raises value error - with tm.assertRaises(ValueError): + with pytest.raises(ValueError): gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), if_exists='xxxxx', private_key=_get_private_key_path()) @@ -1114,7 +1114,7 @@ def test_google_upload_errors_should_raise_exception(self): 'times': [test_timestamp, test_timestamp]}, index=range(2)) - with tm.assertRaises(gbq.StreamingInsertError): + with pytest.raises(gbq.StreamingInsertError): gbq.to_gbq(bad_df, self.destination_table + test_id, _get_project_id(), private_key=_get_private_key_path()) From c6ad8e5251f5abbc20d538766e765d834b74dc7f Mon Sep 17 00:00:00 2001 From: mr-mcox Date: Tue, 13 Jun 2017 12:08:24 -0500 Subject: [PATCH 066/519] When appending to a table, load if the dataframe contains a subset of the existing schema (#24) * Improvements discused in PR conversation Accidentally left a duplicate test in Correcting change to schema made by auto-rebase Fixing missing assertTrue and reversion to not checking subset on append (both from rebase) Replacing AssertEqual Shortening line to pass flake * Making updates per jreback's requested changes * Fixing trailing whitespace * Adding detail to changelog * Use wait_for_job rather than sleep * Revert "Use wait_for_job rather than sleep" This reverts commit 8726a012fb982add846ad7b067aff9c2b1ee68e1. * Minor tweaks before merging * Update the to_gbq doc-string as suggested by @jreback * Make travis happy --- packages/pandas-gbq/docs/source/changelog.rst | 2 + packages/pandas-gbq/docs/source/writing.rst | 2 +- packages/pandas-gbq/pandas_gbq/gbq.py | 92 +++++++++++++++++-- .../pandas-gbq/pandas_gbq/tests/test_gbq.py | 85 +++++++++++++++++ 4 files changed, 171 insertions(+), 10 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 53c1a1b94d1a..011a65a2ebb1 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -6,6 +6,8 @@ Changelog - Resolve issue where the optional ``--noauth_local_webserver`` command line argument would not be propagated during the authentication process. (:issue:`35`) - Drop support for Python 3.4 (:issue:`40`) +- The dataframe passed to ```.to_gbq(...., if_exists='append')``` needs to contain only a subset of the fields in the BigQuery schema. (:issue:`24`) + 0.1.6 / 2017-05-03 ------------------ diff --git a/packages/pandas-gbq/docs/source/writing.rst b/packages/pandas-gbq/docs/source/writing.rst index f0dc0aaa5182..2a30bc3550f8 100644 --- a/packages/pandas-gbq/docs/source/writing.rst +++ b/packages/pandas-gbq/docs/source/writing.rst @@ -40,7 +40,7 @@ a ``TableCreationError`` if the destination table already exists. If the ``if_exists`` argument is set to ``'append'``, the destination dataframe will be written to the table using the defined table schema and column types. The - dataframe must match the destination table in structure and data types. + dataframe must contain fields (matching name and type) currently in the destination table. If the ``if_exists`` argument is set to ``'replace'``, and the existing table has a different schema, a delay of 2 minutes will be forced to ensure that the new schema has propagated in the Google environment. See diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 8d3891003121..0c34124ca5e3 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -557,7 +557,25 @@ def load_data(self, dataframe, dataset_id, table_id, chunksize): self._print("\n") - def verify_schema(self, dataset_id, table_id, schema): + def schema(self, dataset_id, table_id): + """Retrieve the schema of the table + + Obtain from BigQuery the field names and field types + for the table defined by the parameters + + Parameters + ---------- + dataset_id : str + Name of the BigQuery dataset for the table + table_id : str + Name of the BigQuery table + + Returns + ------- + list of dicts + Fields representing the schema + """ + try: from googleapiclient.errors import HttpError except: @@ -573,15 +591,67 @@ def verify_schema(self, dataset_id, table_id, schema): 'type': field_remote['type']} for field_remote in remote_schema['fields']] - fields_remote = set([json.dumps(field_remote) - for field_remote in remote_fields]) - fields_local = set(json.dumps(field_local) - for field_local in schema['fields']) - - return fields_remote == fields_local + return remote_fields except HttpError as ex: self.process_http_error(ex) + def verify_schema(self, dataset_id, table_id, schema): + """Indicate whether schemas match exactly + + Compare the BigQuery table identified in the parameters with + the schema passed in and indicate whether all fields in the former + are present in the latter. Order is not considered. + + Parameters + ---------- + dataset_id :str + Name of the BigQuery dataset for the table + table_id : str + Name of the BigQuery table + schema : list(dict) + Schema for comparison. Each item should have + a 'name' and a 'type' + + Returns + ------- + bool + Whether the schemas match + """ + + fields_remote = sorted(self.schema(dataset_id, table_id), + key=lambda x: x['name']) + fields_local = sorted(schema['fields'], key=lambda x: x['name']) + + return fields_remote == fields_local + + def schema_is_subset(self, dataset_id, table_id, schema): + """Indicate whether the schema to be uploaded is a subset + + Compare the BigQuery table identified in the parameters with + the schema passed in and indicate whether a subset of the fields in + the former are present in the latter. Order is not considered. + + Parameters + ---------- + dataset_id : str + Name of the BigQuery dataset for the table + table_id : str + Name of the BigQuery table + schema : list(dict) + Schema for comparison. Each item should have + a 'name' and a 'type' + + Returns + ------- + bool + Whether the passed schema is a subset + """ + + fields_remote = self.schema(dataset_id, table_id) + fields_local = schema['fields'] + + return all(field in fields_remote for field in fields_local) + def delete_and_recreate_table(self, dataset_id, table_id, table_schema): delay = 0 @@ -810,7 +880,9 @@ def to_gbq(dataframe, destination_table, project_id, chunksize=10000, if_exists : {'fail', 'replace', 'append'}, default 'fail' 'fail': If table exists, do nothing. 'replace': If table exists, drop it, recreate it, and insert data. - 'append': If table exists, insert data. Create if does not exist. + 'append': If table exists and the dataframe schema is a subset of + the destination table schema, insert data. Create destination table + if does not exist. private_key : str (optional) Service account private key in JSON format. Can be file path or string contents. This is useful for remote server @@ -844,7 +916,9 @@ def to_gbq(dataframe, destination_table, project_id, chunksize=10000, connector.delete_and_recreate_table( dataset_id, table_id, table_schema) elif if_exists == 'append': - if not connector.verify_schema(dataset_id, table_id, table_schema): + if not connector.schema_is_subset(dataset_id, + table_id, + table_schema): raise InvalidSchema("Please verify that the structure and " "data types in the DataFrame match the " "schema of the destination table.") diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py index 2ca57b1686a1..069bc7ee3e06 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -1071,6 +1071,31 @@ def test_upload_data_if_table_exists_append(self): _get_project_id(), if_exists='append', private_key=_get_private_key_path()) + def test_upload_subset_columns_if_table_exists_append(self): + # Issue 24: Upload is succesful if dataframe has columns + # which are a subset of the current schema + test_id = "16" + test_size = 10 + df = make_mixed_dataframe_v2(test_size) + df_subset_cols = df.iloc[:, :2] + + # Initialize table with sample data + gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), + chunksize=10000, private_key=_get_private_key_path()) + + # Test the if_exists parameter with value 'append' + gbq.to_gbq(df_subset_cols, + self.destination_table + test_id, _get_project_id(), + if_exists='append', private_key=_get_private_key_path()) + + sleep(30) # <- Curses Google!!! + + result = gbq.read_gbq("SELECT COUNT(*) AS num_rows FROM {0}" + .format(self.destination_table + test_id), + project_id=_get_project_id(), + private_key=_get_private_key_path()) + assert result['num_rows'][0] == test_size * 2 + def test_upload_data_if_table_exists_replace(self): test_id = "4" test_size = 10 @@ -1258,6 +1283,66 @@ def test_verify_schema_ignores_field_mode(self): assert self.sut.verify_schema( self.dataset_prefix + "1", TABLE_ID + test_id, test_schema_2) + def test_retrieve_schema(self): + # Issue #24 schema function returns the schema in biquery + test_id = "15" + test_schema = {'fields': [{'name': 'A', 'type': 'FLOAT'}, + {'name': 'B', 'type': 'FLOAT'}, + {'name': 'C', 'type': 'STRING'}, + {'name': 'D', 'type': 'TIMESTAMP'}]} + + self.table.create(TABLE_ID + test_id, test_schema) + actual = self.sut.schema(self.dataset_prefix + "1", TABLE_ID + test_id) + expected = test_schema['fields'] + assert expected == actual, 'Expected schema used to create table' + + def test_schema_is_subset_passes_if_subset(self): + # Issue #24 schema_is_subset indicates whether the schema of the + # dataframe is a subset of the schema of the bigquery table + test_id = '16' + + table_name = TABLE_ID + test_id + dataset = self.dataset_prefix + '1' + + table_schema = {'fields': [{'name': 'A', + 'type': 'FLOAT'}, + {'name': 'B', + 'type': 'FLOAT'}, + {'name': 'C', + 'type': 'STRING'}]} + tested_schema = {'fields': [{'name': 'A', + 'type': 'FLOAT'}, + {'name': 'B', + 'type': 'FLOAT'}]} + + self.table.create(table_name, table_schema) + + assert self.sut.schema_is_subset( + dataset, table_name, tested_schema) is True + + def test_schema_is_subset_fails_if_not_subset(self): + # For pull request #24 + test_id = '17' + + table_name = TABLE_ID + test_id + dataset = self.dataset_prefix + '1' + + table_schema = {'fields': [{'name': 'A', + 'type': 'FLOAT'}, + {'name': 'B', + 'type': 'FLOAT'}, + {'name': 'C', + 'type': 'STRING'}]} + tested_schema = {'fields': [{'name': 'A', + 'type': 'FLOAT'}, + {'name': 'C', + 'type': 'FLOAT'}]} + + self.table.create(table_name, table_schema) + + assert self.sut.schema_is_subset( + dataset, table_name, tested_schema) is False + def test_list_dataset(self): dataset_id = self.dataset_prefix + "1" assert dataset_id in self.dataset.datasets() From 40535c732e05822e066285d0c2da6a94ffa7edd2 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 16 Jun 2017 09:36:18 -0700 Subject: [PATCH 067/519] BUG: oauth2client deprecated, use google-auth instead. (#39) * BUG: oauth2client deprecated, use google-auth instead. Remove the use of oauth2client and use google-auth library, instead. See GH#37. Rather than check for multiple versions of the libraries, use the setup.py to specify compatible versions. I believe this is safe since Pandas checks for the pandas_gbq package. Since google-auth does not use the argparse module to override user authentication flow settings, add a parameter to choose between the web and console flow. Addresses some eventual consistency issues in table/dataset listing in the integration tests. * MAINT: pandas.util.testing.assertRaises removed This method was removed in https://github.com/pandas-dev/pandas/pull/16089 in favor of pytest.raises. * MAINT: pandas.util.testing.assert_equals removed This method was removed in https://github.com/pandas-dev/pandas/pull/16017 in favor of pytest.raises. * DOC: add version tags for new auth_local_webserver params. * CLN: share _test_imports between main module and tests * TST: pin versions on 3.5 rather than 2.7. --- packages/pandas-gbq/.gitignore | 4 + .../pandas-gbq/ci/requirements-2.7-0.19.2.pip | 10 +- .../pandas-gbq/ci/requirements-3.5-0.18.1.pip | 8 +- .../pandas-gbq/ci/requirements-3.6-0.20.1.pip | 6 +- .../pandas-gbq/ci/requirements-3.6-MASTER.pip | 6 +- packages/pandas-gbq/docs/source/changelog.rst | 4 +- packages/pandas-gbq/pandas_gbq/gbq.py | 333 ++++++++++++------ .../pandas-gbq/pandas_gbq/tests/test_gbq.py | 177 +++------- packages/pandas-gbq/requirements.txt | 4 +- packages/pandas-gbq/setup.py | 11 +- 10 files changed, 321 insertions(+), 242 deletions(-) diff --git a/packages/pandas-gbq/.gitignore b/packages/pandas-gbq/.gitignore index eb19ab7b2a1e..deba4dd80388 100644 --- a/packages/pandas-gbq/.gitignore +++ b/packages/pandas-gbq/.gitignore @@ -76,3 +76,7 @@ Thumbs.db # caches # .cache + +# Credentials # +############### +bigquery_credentials.dat diff --git a/packages/pandas-gbq/ci/requirements-2.7-0.19.2.pip b/packages/pandas-gbq/ci/requirements-2.7-0.19.2.pip index 103055ba7340..852dc1536035 100644 --- a/packages/pandas-gbq/ci/requirements-2.7-0.19.2.pip +++ b/packages/pandas-gbq/ci/requirements-2.7-0.19.2.pip @@ -1,5 +1,7 @@ -httplib2 -google-api-python-client==1.2 -python-gflags==2.0 -oauth2client==1.5.0 +google-api-python-client +google-auth +google-auth-httplib2 +google-auth-oauthlib PyCrypto +python-gflags +mock diff --git a/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip b/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip index 05c938abcbab..6fb8a03d81fc 100644 --- a/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip +++ b/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip @@ -1,3 +1,5 @@ -httplib2 -google-api-python-client -oauth2client +google-api-python-client==1.6.0 +google-auth==1.0.0 +google-auth-httplib2==0.0.1 +google-auth-oauthlib==0.0.1 +mock diff --git a/packages/pandas-gbq/ci/requirements-3.6-0.20.1.pip b/packages/pandas-gbq/ci/requirements-3.6-0.20.1.pip index 05c938abcbab..a1608720f3c6 100644 --- a/packages/pandas-gbq/ci/requirements-3.6-0.20.1.pip +++ b/packages/pandas-gbq/ci/requirements-3.6-0.20.1.pip @@ -1,3 +1,5 @@ -httplib2 google-api-python-client -oauth2client +google-auth +google-auth-httplib2 +google-auth-oauthlib +mock diff --git a/packages/pandas-gbq/ci/requirements-3.6-MASTER.pip b/packages/pandas-gbq/ci/requirements-3.6-MASTER.pip index 05c938abcbab..a1608720f3c6 100644 --- a/packages/pandas-gbq/ci/requirements-3.6-MASTER.pip +++ b/packages/pandas-gbq/ci/requirements-3.6-MASTER.pip @@ -1,3 +1,5 @@ -httplib2 google-api-python-client -oauth2client +google-auth +google-auth-httplib2 +google-auth-oauthlib +mock diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 011a65a2ebb1..05981843430b 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -4,10 +4,10 @@ Changelog 0.2.0 / 2017-??-?? ------------------ -- Resolve issue where the optional ``--noauth_local_webserver`` command line argument would not be propagated during the authentication process. (:issue:`35`) - Drop support for Python 3.4 (:issue:`40`) - The dataframe passed to ```.to_gbq(...., if_exists='append')``` needs to contain only a subset of the fields in the BigQuery schema. (:issue:`24`) - +- Use the `google-auth `__ library for authentication because oauth2client is deprecated. (:issue:`39`) +- ``read_gbq`` now has a ``auth_local_webserver`` boolean argument for controlling whether to use web server or console flow when getting user credentials. Replaces `--noauth_local_webserver` command line argument (:issue:`35`) 0.1.6 / 2017-05-03 ------------------ diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 0c34124ca5e3..b9bb94981c82 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -21,19 +21,18 @@ def _check_google_client_version(): except ImportError: raise ImportError('Could not import pkg_resources (setuptools).') - if compat.PY3: - google_api_minimum_version = '1.4.1' - else: - google_api_minimum_version = '1.2.0' + # Version 1.6.0 is the first version to support google-auth. + # https://github.com/google/google-api-python-client/blob/master/CHANGELOG + google_api_minimum_version = '1.6.0' _GOOGLE_API_CLIENT_VERSION = pkg_resources.get_distribution( 'google-api-python-client').version if (StrictVersion(_GOOGLE_API_CLIENT_VERSION) < StrictVersion(google_api_minimum_version)): - raise ImportError("pandas requires google-api-python-client >= {0} " - "for Google BigQuery support, " - "current version {1}" + raise ImportError('pandas requires google-api-python-client >= {0} ' + 'for Google BigQuery support, ' + 'current version {1}' .format(google_api_minimum_version, _GOOGLE_API_CLIENT_VERSION)) @@ -42,19 +41,64 @@ def _test_google_api_imports(): try: import httplib2 # noqa - try: - from googleapiclient.discovery import build # noqa - from googleapiclient.errors import HttpError # noqa - except: - from apiclient.discovery import build # noqa - from apiclient.errors import HttpError # noqa - from oauth2client.client import AccessTokenRefreshError # noqa - from oauth2client.client import OAuth2WebServerFlow # noqa - from oauth2client.file import Storage # noqa - from oauth2client.tools import run_flow, argparser # noqa - except ImportError as e: - raise ImportError("Missing module required for Google BigQuery " - "support: {0}".format(str(e))) + except ImportError as ex: + raise ImportError( + 'pandas requires httplib2 for Google BigQuery support: ' + '{0}'.format(ex)) + + try: + from google_auth_oauthlib.flow import InstalledAppFlow # noqa + except ImportError as ex: + raise ImportError( + 'pandas requires google-auth-oauthlib for Google BigQuery ' + 'support: {0}'.format(ex)) + + try: + from google_auth_httplib2 import AuthorizedHttp # noqa + from google_auth_httplib2 import Request # noqa + except ImportError as ex: + raise ImportError( + 'pandas requires google-auth-httplib2 for Google BigQuery ' + 'support: {0}'.format(ex)) + + try: + from googleapiclient.discovery import build # noqa + from googleapiclient.errors import HttpError # noqa + except ImportError as ex: + raise ImportError( + "pandas requires google-api-python-client for Google BigQuery " + "support: {0}".format(ex)) + + try: + import google.auth # noqa + except ImportError as ex: + raise ImportError( + "pandas requires google-auth for Google BigQuery support: " + "{0}".format(ex)) + + _check_google_client_version() + + +def _try_credentials(project_id, credentials): + import httplib2 + from googleapiclient.discovery import build + import googleapiclient.errors + from google_auth_httplib2 import AuthorizedHttp + + if credentials is None: + return None + + http = httplib2.Http() + try: + authed_http = AuthorizedHttp(credentials, http=http) + bigquery_service = build('bigquery', 'v2', http=authed_http) + # Check if the application has rights to the BigQuery project + jobs = bigquery_service.jobs() + job_data = {'configuration': {'query': {'query': 'SELECT 1'}}} + jobs.insert(projectId=project_id, body=job_data).execute() + return credentials + except googleapiclient.errors.Error: + return None class InvalidPrivateKeyFormat(ValueError): @@ -147,13 +191,13 @@ class GbqConnector(object): scope = 'https://www.googleapis.com/auth/bigquery' def __init__(self, project_id, reauth=False, verbose=False, - private_key=None, dialect='legacy'): - _check_google_client_version() - _test_google_api_imports() + private_key=None, auth_local_webserver=False, + dialect='legacy'): self.project_id = project_id self.reauth = reauth self.verbose = verbose self.private_key = private_key + self.auth_local_webserver = auth_local_webserver self.dialect = dialect self.credentials = self.get_credentials() self.service = self.get_service() @@ -188,78 +232,134 @@ def get_application_default_credentials(self): from the environment. Or, the retrieved credentials do not have access to the project (self.project_id) on BigQuery. """ - import httplib2 - try: - from googleapiclient.discovery import build - except ImportError: - from apiclient.discovery import build + import google.auth + from google.auth.exceptions import DefaultCredentialsError + try: - from oauth2client.client import GoogleCredentials - except ImportError: + credentials, _ = google.auth.default(scopes=[self.scope]) + except (DefaultCredentialsError, IOError): return None + return _try_credentials(self.project_id, credentials) + + def load_user_account_credentials(self): + """ + Loads user account credentials from a local file. + + .. versionadded 0.2.0 + + Parameters + ---------- + None + + Returns + ------- + - GoogleCredentials, + If the credentials can loaded. The retrieved credentials should + also have access to the project (self.project_id) on BigQuery. + - OR None, + If credentials can not be loaded from a file. Or, the retrieved + credentials do not have access to the project (self.project_id) + on BigQuery. + """ + import httplib2 + from google_auth_httplib2 import Request + from google.oauth2.credentials import Credentials + try: - credentials = GoogleCredentials.get_application_default() - except: + with open('bigquery_credentials.dat') as credentials_file: + credentials_json = json.load(credentials_file) + except (IOError, ValueError): return None + credentials = Credentials( + token=credentials_json.get('access_token'), + refresh_token=credentials_json.get('refresh_token'), + id_token=credentials_json.get('id_token'), + token_uri=credentials_json.get('token_uri'), + client_id=credentials_json.get('client_id'), + client_secret=credentials_json.get('client_secret'), + scopes=credentials_json.get('scopes')) + + # Refresh the token before trying to use it. http = httplib2.Http() + request = Request(http) + credentials.refresh(request) + + return _try_credentials(self.project_id, credentials) + + def save_user_account_credentials(self, credentials): + """ + Saves user account credentials to a local file. + + .. versionadded 0.2.0 + """ try: - http = credentials.authorize(http) - bigquery_service = build('bigquery', 'v2', http=http) - # Check if the application has rights to the BigQuery project - jobs = bigquery_service.jobs() - job_data = {'configuration': {'query': {'query': 'SELECT 1'}}} - jobs.insert(projectId=self.project_id, body=job_data).execute() - return credentials - except: - return None + with open('bigquery_credentials.dat', 'w') as credentials_file: + credentials_json = { + 'refresh_token': credentials.refresh_token, + 'id_token': credentials.id_token, + 'token_uri': credentials.token_uri, + 'client_id': credentials.client_id, + 'client_secret': credentials.client_secret, + 'scopes': credentials.scopes, + } + json.dump(credentials_json, credentials_file) + except IOError: + self._print('Unable to save credentials.') def get_user_account_credentials(self): - from oauth2client.client import OAuth2WebServerFlow - from oauth2client.file import Storage - from oauth2client.tools import run_flow, argparser + """Gets user account credentials. - flow = OAuth2WebServerFlow( - client_id=('495642085510-k0tmvj2m941jhre2nbqka17vqpjfddtd' - '.apps.googleusercontent.com'), - client_secret='kOc9wMptUtxkcIFbtZCcrEAc', - scope=self.scope, - redirect_uri='urn:ietf:wg:oauth:2.0:oob') + This method authenticates using user credentials, either loading saved + credentials from a file or by going through the OAuth flow. - storage = Storage('bigquery_credentials.dat') - credentials = storage.get() - - if credentials is None or credentials.invalid or self.reauth: - credentials = run_flow(flow, storage, argparser.parse_args()) + Parameters + ---------- + None - return credentials + Returns + ------- + GoogleCredentials : credentials + Credentials for the user with BigQuery access. + """ + from google_auth_oauthlib.flow import InstalledAppFlow + from oauthlib.oauth2.rfc6749.errors import OAuth2Error + + credentials = self.load_user_account_credentials() + + client_config = { + 'installed': { + 'client_id': ('495642085510-k0tmvj2m941jhre2nbqka17vqpjfddtd' + '.apps.googleusercontent.com'), + 'client_secret': 'kOc9wMptUtxkcIFbtZCcrEAc', + 'redirect_uris': ['urn:ietf:wg:oauth:2.0:oob'], + 'auth_uri': 'https://accounts.google.com/o/oauth2/auth', + 'token_uri': 'https://accounts.google.com/o/oauth2/token', + } + } - def get_service_account_credentials(self): - # Bug fix for https://github.com/pandas-dev/pandas/issues/12572 - # We need to know that a supported version of oauth2client is installed - # Test that either of the following is installed: - # - SignedJwtAssertionCredentials from oauth2client.client - # - ServiceAccountCredentials from oauth2client.service_account - # SignedJwtAssertionCredentials is available in oauthclient < 2.0.0 - # ServiceAccountCredentials is available in oauthclient >= 2.0.0 - oauth2client_v1 = True - oauth2client_v2 = True + if credentials is None or self.reauth: + app_flow = InstalledAppFlow.from_client_config( + client_config, scopes=[self.scope]) - try: - from oauth2client.client import SignedJwtAssertionCredentials - except ImportError: - oauth2client_v1 = False + try: + if self.auth_local_webserver: + credentials = app_flow.run_local_server() + else: + credentials = app_flow.run_console() + except OAuth2Error as ex: + raise AccessDenied( + "Unable to get valid credentials: {0}".format(ex)) - try: - from oauth2client.service_account import ServiceAccountCredentials - except ImportError: - oauth2client_v2 = False + self.save_user_account_credentials(credentials) - if not oauth2client_v1 and not oauth2client_v2: - raise ImportError("Missing oauth2client required for BigQuery " - "service account support") + return credentials + def get_service_account_credentials(self): + import httplib2 + from google_auth_httplib2 import Request + from google.oauth2.service_account import Credentials from os.path import isfile try: @@ -277,16 +377,15 @@ def get_service_account_credentials(self): json_key['private_key'] = bytes( json_key['private_key'], 'UTF-8') - if oauth2client_v1: - return SignedJwtAssertionCredentials( - json_key['client_email'], - json_key['private_key'], - self.scope, - ) - else: - return ServiceAccountCredentials.from_json_keyfile_dict( - json_key, - self.scope) + credentials = Credentials.from_service_account_info(json_key) + credentials = credentials.with_scopes([self.scope]) + + # Refresh the token before trying to use it. + http = httplib2.Http() + request = Request(http) + credentials.refresh(request) + + return credentials except (KeyError, ValueError, TypeError, AttributeError): raise InvalidPrivateKeyFormat( "Private key is missing or invalid. It should be service " @@ -324,14 +423,13 @@ def sizeof_fmt(num, suffix='B'): def get_service(self): import httplib2 - try: - from googleapiclient.discovery import build - except: - from apiclient.discovery import build + from google_auth_httplib2 import AuthorizedHttp + from googleapiclient.discovery import build http = httplib2.Http() - http = self.credentials.authorize(http) - bigquery_service = build('bigquery', 'v2', http=http) + authed_http = AuthorizedHttp( + self.credentials, http=http) + bigquery_service = build('bigquery', 'v2', http=authed_http) return bigquery_service @@ -380,9 +478,7 @@ def run_query(self, query, **kwargs): from googleapiclient.errors import HttpError except: from apiclient.errors import HttpError - from oauth2client.client import AccessTokenRefreshError - - _check_google_client_version() + from google.auth.exceptions import RefreshError job_collection = self.service.jobs() @@ -419,7 +515,7 @@ def run_query(self, query, **kwargs): query_reply = job_collection.insert( projectId=self.project_id, body=job_data).execute() self._print('ok.\nQuery running...') - except (AccessTokenRefreshError, ValueError): + except (RefreshError, ValueError): if self.private_key: raise AccessDenied( "The service account credentials are not valid") @@ -711,8 +807,8 @@ def _parse_entry(field_value, field_type): def read_gbq(query, project_id=None, index_col=None, col_order=None, - reauth=False, verbose=True, private_key=None, dialect='legacy', - **kwargs): + reauth=False, verbose=True, private_key=None, + auth_local_webserver=False, dialect='legacy', **kwargs): r"""Load data from Google BigQuery. The main method a user calls to execute a Query in Google BigQuery @@ -756,6 +852,15 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, Service account private key in JSON format. Can be file path or string contents. This is useful for remote server authentication (eg. jupyter iPython notebook on remote host) + auth_local_webserver : boolean, default False + Use the [local webserver flow] instead of the [console flow] when + getting user credentials. + + .. [local webserver flow] + http://google-auth-oauthlib.readthedocs.io/en/latest/reference/google_auth_oauthlib.flow.html#google_auth_oauthlib.flow.InstalledAppFlow.run_local_server + .. [console flow] + http://google-auth-oauthlib.readthedocs.io/en/latest/reference/google_auth_oauthlib.flow.html#google_auth_oauthlib.flow.InstalledAppFlow.run_console + .. versionadded:: 0.2.0 dialect : {'legacy', 'standard'}, default 'legacy' 'legacy' : Use BigQuery's legacy SQL dialect. @@ -780,15 +885,17 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, """ + _test_google_api_imports() + if not project_id: raise TypeError("Missing required parameter: project_id") if dialect not in ('legacy', 'standard'): raise ValueError("'{0}' is not valid for dialect".format(dialect)) - connector = GbqConnector(project_id, reauth=reauth, verbose=verbose, - private_key=private_key, - dialect=dialect) + connector = GbqConnector( + project_id, reauth=reauth, verbose=verbose, private_key=private_key, + dialect=dialect, auth_local_webserver=auth_local_webserver) schema, pages = connector.run_query(query, **kwargs) dataframe_list = [] while len(pages) > 0: @@ -838,7 +945,8 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, def to_gbq(dataframe, destination_table, project_id, chunksize=10000, - verbose=True, reauth=False, if_exists='fail', private_key=None): + verbose=True, reauth=False, if_exists='fail', private_key=None, + auth_local_webserver=False): """Write a DataFrame to a Google BigQuery table. The main method a user calls to export pandas DataFrame contents to @@ -887,8 +995,19 @@ def to_gbq(dataframe, destination_table, project_id, chunksize=10000, Service account private key in JSON format. Can be file path or string contents. This is useful for remote server authentication (eg. jupyter iPython notebook on remote host) + auth_local_webserver : boolean, default False + Use the [local webserver flow] instead of the [console flow] when + getting user credentials. + + .. [local webserver flow] + http://google-auth-oauthlib.readthedocs.io/en/latest/reference/google_auth_oauthlib.flow.html#google_auth_oauthlib.flow.InstalledAppFlow.run_local_server + .. [console flow] + http://google-auth-oauthlib.readthedocs.io/en/latest/reference/google_auth_oauthlib.flow.html#google_auth_oauthlib.flow.InstalledAppFlow.run_console + .. versionadded:: 0.2.0 """ + _test_google_api_imports() + if if_exists not in ('fail', 'replace', 'append'): raise ValueError("'{0}' is not valid for if_exists".format(if_exists)) @@ -896,8 +1015,9 @@ def to_gbq(dataframe, destination_table, project_id, chunksize=10000, raise NotFoundException( "Invalid Table Name. Should be of the form 'datasetId.tableId' ") - connector = GbqConnector(project_id, reauth=reauth, verbose=verbose, - private_key=private_key) + connector = GbqConnector( + project_id, reauth=reauth, verbose=verbose, private_key=private_key, + auth_local_webserver=auth_local_webserver) dataset_id, table_id = destination_table.rsplit('.', 1) table = _Table(project_id, dataset_id, reauth=reauth, @@ -1127,6 +1247,9 @@ def datasets(self): pageToken=next_page_token).execute() dataset_response = list_dataset_response.get('datasets') + if dataset_response is None: + dataset_response = [] + next_page_token = list_dataset_response.get('nextPageToken') if dataset_response is None: diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py index 069bc7ee3e06..e8eda1d3f01a 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -10,7 +10,6 @@ import numpy as np -from distutils.version import StrictVersion from pandas import compat from pandas.compat import u, range @@ -23,13 +22,6 @@ TABLE_ID = 'new_test' -_IMPORTS = False -_GOOGLE_API_CLIENT_INSTALLED = False -_GOOGLE_API_CLIENT_VALID_VERSION = False -_HTTPLIB2_INSTALLED = False -_SETUPTOOLS_INSTALLED = False - - def _skip_if_no_project_id(): if not _get_project_id(): pytest.skip( @@ -84,98 +76,19 @@ def _get_private_key_contents(): def _test_imports(): - global _GOOGLE_API_CLIENT_INSTALLED, _GOOGLE_API_CLIENT_VALID_VERSION, \ - _HTTPLIB2_INSTALLED, _SETUPTOOLS_INSTALLED - try: - import pkg_resources - _SETUPTOOLS_INSTALLED = True + import pkg_resources # noqa except ImportError: - _SETUPTOOLS_INSTALLED = False - - if compat.PY3: - google_api_minimum_version = '1.4.1' - else: - google_api_minimum_version = '1.2.0' - - if _SETUPTOOLS_INSTALLED: - try: - try: - from googleapiclient.discovery import build # noqa - from googleapiclient.errors import HttpError # noqa - except: - from apiclient.discovery import build # noqa - from apiclient.errors import HttpError # noqa - - from oauth2client.client import OAuth2WebServerFlow # noqa - from oauth2client.client import AccessTokenRefreshError # noqa - - from oauth2client.file import Storage # noqa - from oauth2client.tools import run_flow # noqa - _GOOGLE_API_CLIENT_INSTALLED = True - _GOOGLE_API_CLIENT_VERSION = pkg_resources.get_distribution( - 'google-api-python-client').version - - if (StrictVersion(_GOOGLE_API_CLIENT_VERSION) >= - StrictVersion(google_api_minimum_version)): - _GOOGLE_API_CLIENT_VALID_VERSION = True - - except ImportError: - _GOOGLE_API_CLIENT_INSTALLED = False - - try: - import httplib2 # noqa - _HTTPLIB2_INSTALLED = True - except ImportError: - _HTTPLIB2_INSTALLED = False - - if not _SETUPTOOLS_INSTALLED: raise ImportError('Could not import pkg_resources (setuptools).') - if not _GOOGLE_API_CLIENT_INSTALLED: - raise ImportError('Could not import Google API Client.') - - if not _GOOGLE_API_CLIENT_VALID_VERSION: - raise ImportError("pandas requires google-api-python-client >= {0} " - "for Google BigQuery support, " - "current version {1}" - .format(google_api_minimum_version, - _GOOGLE_API_CLIENT_VERSION)) - - if not _HTTPLIB2_INSTALLED: - raise ImportError( - "pandas requires httplib2 for Google BigQuery support") - - # Bug fix for https://github.com/pandas-dev/pandas/issues/12572 - # We need to know that a supported version of oauth2client is installed - # Test that either of the following is installed: - # - SignedJwtAssertionCredentials from oauth2client.client - # - ServiceAccountCredentials from oauth2client.service_account - # SignedJwtAssertionCredentials is available in oauthclient < 2.0.0 - # ServiceAccountCredentials is available in oauthclient >= 2.0.0 - oauth2client_v1 = True - oauth2client_v2 = True - - try: - from oauth2client.client import SignedJwtAssertionCredentials # noqa - except ImportError: - oauth2client_v1 = False - - try: - from oauth2client.service_account import ServiceAccountCredentials # noqa - except ImportError: - oauth2client_v2 = False - - if not oauth2client_v1 and not oauth2client_v2: - raise ImportError("Missing oauth2client required for BigQuery " - "service account support") + gbq._test_google_api_imports() def _setup_common(): try: _test_imports() except (ImportError, NotImplementedError) as import_exception: - pytest.skip(import_exception) + pytest.skip(str(import_exception)) if _in_travis_environment(): logging.getLogger('oauth2client').setLevel(logging.ERROR) @@ -185,26 +98,18 @@ def _setup_common(): def _check_if_can_get_correct_default_credentials(): # Checks if "Application Default Credentials" can be fetched # from the environment the tests are running in. - # See Issue #13577 + # See https://github.com/pandas-dev/pandas/issues/13577 + + import google.auth + from google.auth.exceptions import DefaultCredentialsError - import httplib2 - try: - from googleapiclient.discovery import build - except ImportError: - from apiclient.discovery import build try: - from oauth2client.client import GoogleCredentials - credentials = GoogleCredentials.get_application_default() - http = httplib2.Http() - http = credentials.authorize(http) - bigquery_service = build('bigquery', 'v2', http=http) - jobs = bigquery_service.jobs() - job_data = {'configuration': {'query': {'query': 'SELECT 1'}}} - jobs.insert(projectId=_get_project_id(), body=job_data).execute() - return True - except: + credentials, _ = google.auth.default(scopes=[gbq.GbqConnector.scope]) + except (DefaultCredentialsError, IOError): return False + return gbq._try_credentials(_get_project_id(), credentials) is not None + def clean_gbq_environment(dataset_prefix, private_key=None): dataset = gbq._Dataset(_get_project_id(), private_key=private_key) @@ -219,17 +124,31 @@ def clean_gbq_environment(dataset_prefix, private_key=None): if dataset_id in all_datasets: table = gbq._Table(_get_project_id(), dataset_id, private_key=private_key) + + # Table listing is eventually consistent, so loop until + # all tables no longer appear (max 30 seconds). + table_retry = 30 all_tables = dataset.tables(dataset_id) - for table_id in all_tables: - table.delete(table_id) + while all_tables and table_retry > 0: + for table_id in all_tables: + try: + table.delete(table_id) + except gbq.NotFoundException: + pass + sleep(1) + table_retry = table_retry - 1 + all_tables = dataset.tables(dataset_id) dataset.delete(dataset_id) retry = 0 except gbq.GenericGBQException as ex: - # Build in retry logic to work around the following error : + # Build in retry logic to work around the following errors : # An internal error occurred and the request could not be... - if 'An internal error occurred' in ex.message and retry > 0: - pass + # Dataset ... is still in use + error_message = str(ex).lower() + if ('an internal error occurred' in error_message or + 'still in use' in error_message) and retry > 0: + sleep(30) else: raise ex @@ -264,14 +183,15 @@ def setup_method(self, method): _skip_if_no_project_id() _skip_local_auth_if_in_travis_env() - self.sut = gbq.GbqConnector(_get_project_id()) + self.sut = gbq.GbqConnector( + _get_project_id(), auth_local_webserver=True) def test_should_be_able_to_make_a_connector(self): assert self.sut is not None, 'Could not create a GbqConnector' def test_should_be_able_to_get_valid_credentials(self): credentials = self.sut.get_credentials() - assert credentials.invalid != 'Returned credentials invalid' + assert credentials.valid def test_should_be_able_to_get_a_bigquery_service(self): bigquery_service = self.sut.get_service() @@ -287,18 +207,35 @@ def test_should_be_able_to_get_results_from_query(self): def test_get_application_default_credentials_does_not_throw_error(self): if _check_if_can_get_correct_default_credentials(): - pytest.skip("Can get default_credentials " - "from the environment!") - credentials = self.sut.get_application_default_credentials() + # Can get real credentials, so mock it out to fail. + import mock + from google.auth.exceptions import DefaultCredentialsError + with mock.patch('google.auth.default', + side_effect=DefaultCredentialsError()): + credentials = self.sut.get_application_default_credentials() + else: + credentials = self.sut.get_application_default_credentials() assert credentials is None def test_get_application_default_credentials_returns_credentials(self): if not _check_if_can_get_correct_default_credentials(): pytest.skip("Cannot get default_credentials " "from the environment!") - from oauth2client.client import GoogleCredentials + from google.auth.credentials import Credentials credentials = self.sut.get_application_default_credentials() - assert isinstance(credentials, GoogleCredentials) + assert isinstance(credentials, Credentials) + + def test_get_user_account_credentials_bad_file_returns_credentials(self): + import mock + from google.auth.credentials import Credentials + with mock.patch('__main__.open', side_effect=IOError()): + credentials = self.sut.get_user_account_credentials() + assert isinstance(credentials, Credentials) + + def test_get_user_account_credentials_returns_credentials(self): + from google.auth.credentials import Credentials + credentials = self.sut.get_user_account_credentials() + assert isinstance(credentials, Credentials) class TestGBQConnectorIntegrationWithServiceAccountKeyPath(object): @@ -317,7 +254,7 @@ def test_should_be_able_to_make_a_connector(self): def test_should_be_able_to_get_valid_credentials(self): credentials = self.sut.get_credentials() - assert not credentials.invalid + assert credentials.valid def test_should_be_able_to_get_a_bigquery_service(self): bigquery_service = self.sut.get_service() @@ -348,7 +285,7 @@ def test_should_be_able_to_make_a_connector(self): def test_should_be_able_to_get_valid_credentials(self): credentials = self.sut.get_credentials() - assert not credentials.invalid + assert credentials.valid def test_should_be_able_to_get_a_bigquery_service(self): bigquery_service = self.sut.get_service() diff --git a/packages/pandas-gbq/requirements.txt b/packages/pandas-gbq/requirements.txt index 11bb601852d4..c72b5a5a7de2 100644 --- a/packages/pandas-gbq/requirements.txt +++ b/packages/pandas-gbq/requirements.txt @@ -1,4 +1,6 @@ pandas httplib2 google-api-python-client -oauth2client +google-auth +google-auth-httplib2 +google-auth-oauthlib diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index a3b8f06f7ed8..df3cd85d1ee4 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -17,9 +17,14 @@ def readme(): return f.read() -INSTALL_REQUIRES = ( - ['pandas', 'httplib2', 'google-api-python-client', 'oauth2client'] -) +INSTALL_REQUIRES = [ + 'pandas', + 'httplib2>=0.9.2', + 'google-api-python-client>=1.6.0', + 'google-auth>=1.0.0', + 'google-auth-httplib2>=0.0.1', + 'google-auth-oauthlib>=0.0.1', +] setup( From 021734c2306642f72f43d10d5dcb5f5e13c4e6da Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 22 Jun 2017 19:16:32 -0700 Subject: [PATCH 068/519] DOC: remove references to oauth2client (#61) Closes #59. --- packages/pandas-gbq/docs/source/install.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/docs/source/install.rst b/packages/pandas-gbq/docs/source/install.rst index 5b0490f87ef1..2b701fd2443f 100644 --- a/packages/pandas-gbq/docs/source/install.rst +++ b/packages/pandas-gbq/docs/source/install.rst @@ -39,4 +39,6 @@ This module requires following additional dependencies: - `httplib2 `__: HTTP client - `google-api-python-client `__: Google's API client -- `oauth2client `__: authentication and authorization for Google's API +- `google-auth `__: authentication and authorization for Google's API +- `google-auth-oauthlib `__: integration with `oauthlib `__ for end-user authentication +- `google-auth-httplib2 `__: adapter to use ``httplib2`` HTTP client with ``google-auth`` From e781b2cc54dcbb9f6d45b32018bde1c15a32a915 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 27 Jun 2017 21:15:14 -0700 Subject: [PATCH 069/519] TST: Install dependencies with Conda on py3.6 (#62) Cover dependency installation via Conda in one of the tests. --- packages/pandas-gbq/.travis.yml | 13 +++++++++---- ...3.6-0.20.1.pip => requirements-3.6-0.20.1.conda} | 0 2 files changed, 9 insertions(+), 4 deletions(-) rename packages/pandas-gbq/ci/{requirements-3.6-0.20.1.pip => requirements-3.6-0.20.1.conda} (100%) diff --git a/packages/pandas-gbq/.travis.yml b/packages/pandas-gbq/.travis.yml index 881974df4998..9beb94236515 100644 --- a/packages/pandas-gbq/.travis.yml +++ b/packages/pandas-gbq/.travis.yml @@ -5,8 +5,8 @@ language: python env: - PYTHON=2.7 PANDAS=0.19.2 COVERAGE='false' LINT='true' - PYTHON=3.5 PANDAS=0.18.1 COVERAGE='true' LINT='false' - - PYTHON=3.6 PANDAS=0.20.1 COVERAGE='false' LINT='true' - - PYTHON=3.6 PANDAS=MASTER COVERAGE='false' LINT='false' + - PYTHON=3.6 PANDAS=0.20.1 COVERAGE='false' LINT='false' + - PYTHON=3.6 PANDAS=MASTER COVERAGE='false' LINT='true' before_install: - echo "before_install" @@ -19,6 +19,7 @@ install: - hash -r - conda config --set always_yes yes --set changeps1 no - conda config --add channels pandas + - conda config --add channels conda-forge - conda update -q conda - conda info -a - conda create -n test-environment python=$PYTHON @@ -31,8 +32,12 @@ install: conda install pandas=$PANDAS; fi - pip install coverage pytest pytest-cov flake8 codecov - - REQ="ci/requirements-${PYTHON}-${PANDAS}.pip" - - pip install -r $REQ + - REQ="ci/requirements-${PYTHON}-${PANDAS}" + - if [ -f "$REQ.pip" ]; then + pip install -r "$REQ.pip"; + else + conda install --file "$REQ.conda"; + fi - conda list - python setup.py install diff --git a/packages/pandas-gbq/ci/requirements-3.6-0.20.1.pip b/packages/pandas-gbq/ci/requirements-3.6-0.20.1.conda similarity index 100% rename from packages/pandas-gbq/ci/requirements-3.6-0.20.1.pip rename to packages/pandas-gbq/ci/requirements-3.6-0.20.1.conda From c0cb3023dc0137fac49e7528a9696278a38531f5 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 30 Jun 2017 09:25:18 -0700 Subject: [PATCH 070/519] DOC: BigQuery is one word (#65) Fixing a small nit. Also, an interface to an interface sounded weird, so I changed it to BigQuery API. --- packages/pandas-gbq/README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/README.rst b/packages/pandas-gbq/README.rst index 803ad7dffeac..d23565199aa1 100644 --- a/packages/pandas-gbq/README.rst +++ b/packages/pandas-gbq/README.rst @@ -3,7 +3,7 @@ pandas-gbq |Build Status| |Version Status| |Coverage Status| -**pandas-gbq** is a package providing an interface to the Google Big Query interface from pandas +**pandas-gbq** is a package providing an interface to the Google BigQuery API from pandas Installation From bba39030cbc2fb5b44b1c19d8080702f1c9c145e Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Thu, 6 Jul 2017 06:06:00 -0400 Subject: [PATCH 071/519] CI: remove coverage check for project --- packages/pandas-gbq/codecov.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/codecov.yml b/packages/pandas-gbq/codecov.yml index e97acea7d26d..27144a44fa13 100644 --- a/packages/pandas-gbq/codecov.yml +++ b/packages/pandas-gbq/codecov.yml @@ -2,7 +2,7 @@ coverage: status: project: default: - target: '30' + target: null patch: default: target: '50' From 56837086c3cf4830d5ff69ec5114bf00a1f9059e Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 6 Jul 2017 03:19:26 -0700 Subject: [PATCH 072/519] DOC: Add development installation instructions (#64) Includes conda install steps for dependencies. --- .../pandas-gbq/docs/source/contributing.rst | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/packages/pandas-gbq/docs/source/contributing.rst b/packages/pandas-gbq/docs/source/contributing.rst index d3376a30cb08..986e61edff19 100644 --- a/packages/pandas-gbq/docs/source/contributing.rst +++ b/packages/pandas-gbq/docs/source/contributing.rst @@ -139,6 +139,52 @@ request. If you have uncommitted changes, you will need to ``stash`` them prior to updating. This will effectively store your changes and they can be reapplied after updating. +Install in Development Mode +--------------------------- + +It's helpful to install pandas-gbq in development mode so that you can +use the library without reinstalling the package after every change. + +Conda +~~~~~ + +Create a new conda environment and install the necessary dependencies + +.. code-block:: shell + + $ conda create -n my-env --channel conda-forge \ + pandas \ + google-auth-oauthlib \ + google-api-python-client \ + google-auth-httplib2 + $ source activate my-env + +Install pandas-gbq in development mode + +.. code-block:: shell + + $ python setup.py develop + +Pip & virtualenv +~~~~~~~~~~~~~~~~ + +*Skip this section if you already followed the conda instructions.* + +Create a new `virtual +environment `__. + +.. code-block:: shell + + $ virtualenv env + $ source env/bin/activate + +You can install pandas-gbq and its dependencies in `development mode via +pip `__. + +.. code-block:: shell + + $ pip install -e . + Contributing to the code base ============================= From 3c2d236b4b6ad8c416783dad9e48995b48bb4dc3 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Thu, 6 Jul 2017 06:23:38 -0400 Subject: [PATCH 073/519] fix codedcov again --- packages/pandas-gbq/codecov.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/codecov.yml b/packages/pandas-gbq/codecov.yml index 27144a44fa13..62aff1706008 100644 --- a/packages/pandas-gbq/codecov.yml +++ b/packages/pandas-gbq/codecov.yml @@ -2,7 +2,7 @@ coverage: status: project: default: - target: null + target: '0' patch: default: target: '50' From 7cfeb17f7573a39ebf0446d0bb2893cbd97d7ad4 Mon Sep 17 00:00:00 2001 From: Piotr Chromiec Date: Tue, 11 Jul 2017 18:04:29 +0200 Subject: [PATCH 074/519] ENH: BQ Job Id in verbose output (#70) * ENH: BQ Job Id in verbose output * changelog entry --- packages/pandas-gbq/docs/source/changelog.rst | 3 ++- packages/pandas-gbq/pandas_gbq/gbq.py | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 05981843430b..a69bda999d63 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -7,7 +7,8 @@ Changelog - Drop support for Python 3.4 (:issue:`40`) - The dataframe passed to ```.to_gbq(...., if_exists='append')``` needs to contain only a subset of the fields in the BigQuery schema. (:issue:`24`) - Use the `google-auth `__ library for authentication because oauth2client is deprecated. (:issue:`39`) -- ``read_gbq`` now has a ``auth_local_webserver`` boolean argument for controlling whether to use web server or console flow when getting user credentials. Replaces `--noauth_local_webserver` command line argument (:issue:`35`) +- ``read_gbq`` now has a ``auth_local_webserver`` boolean argument for controlling whether to use web server or console flow when getting user credentials. Replaces `--noauth_local_webserver` command line argument. (:issue:`35`) +- ``read_gbq`` now displays the BigQuery Job ID in verbose output. (:issue:`70`) 0.1.6 / 2017-05-03 ------------------ diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index b9bb94981c82..5fa78153ca2d 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -514,7 +514,7 @@ def run_query(self, query, **kwargs): self._print('Requesting query... ', end="") query_reply = job_collection.insert( projectId=self.project_id, body=job_data).execute() - self._print('ok.\nQuery running...') + self._print('ok.') except (RefreshError, ValueError): if self.private_key: raise AccessDenied( @@ -527,13 +527,15 @@ def run_query(self, query, **kwargs): self.process_http_error(ex) job_reference = query_reply['jobReference'] + job_id = job_reference['jobId'] + self._print('Job ID: %s\nQuery running...' % job_id) while not query_reply.get('jobComplete', False): self.print_elapsed_seconds(' Elapsed', 's. Waiting...') try: query_reply = job_collection.getQueryResults( projectId=job_reference['projectId'], - jobId=job_reference['jobId']).execute() + jobId=job_id).execute() except HttpError as ex: self.process_http_error(ex) @@ -584,7 +586,7 @@ def run_query(self, query, **kwargs): try: query_reply = job_collection.getQueryResults( projectId=job_reference['projectId'], - jobId=job_reference['jobId'], + jobId=job_id, pageToken=page_token).execute() except HttpError as ex: self.process_http_error(ex) From 936ae710c14fe61eff6b7059ab7acd5679f69c53 Mon Sep 17 00:00:00 2001 From: Piotr Chromiec Date: Wed, 12 Jul 2017 15:25:57 +0200 Subject: [PATCH 075/519] ENH: query standard price (USD) presentation (#71) --- packages/pandas-gbq/docs/source/changelog.rst | 2 +- packages/pandas-gbq/pandas_gbq/gbq.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index a69bda999d63..b689ceee6099 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -8,7 +8,7 @@ Changelog - The dataframe passed to ```.to_gbq(...., if_exists='append')``` needs to contain only a subset of the fields in the BigQuery schema. (:issue:`24`) - Use the `google-auth `__ library for authentication because oauth2client is deprecated. (:issue:`39`) - ``read_gbq`` now has a ``auth_local_webserver`` boolean argument for controlling whether to use web server or console flow when getting user credentials. Replaces `--noauth_local_webserver` command line argument. (:issue:`35`) -- ``read_gbq`` now displays the BigQuery Job ID in verbose output. (:issue:`70`) +- ``read_gbq`` now displays the BigQuery Job ID and standard price in verbose output. (:issue:`70` and :issue:`71`) 0.1.6 / 2017-05-03 ------------------ diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 5fa78153ca2d..452de5ef2331 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -202,6 +202,10 @@ def __init__(self, project_id, reauth=False, verbose=False, self.credentials = self.get_credentials() self.service = self.get_service() + # BQ Queries costs $5 per TB. First 1 TB per month is free + # see here for more: https://cloud.google.com/bigquery/pricing + self.query_price_for_TB = 5. / 2**40 # USD/TB + def get_credentials(self): if self.private_key: return self.get_service_account_credentials() @@ -545,8 +549,10 @@ def run_query(self, query, **kwargs): else: bytes_processed = int(query_reply.get( 'totalBytesProcessed', '0')) - self._print('Query done.\nProcessed: {}\n'.format( + self._print('Query done.\nProcessed: {}'.format( self.sizeof_fmt(bytes_processed))) + self._print('Standard price: ${:,.2f} USD\n'.format( + bytes_processed * self.query_price_for_TB)) self._print('Retrieving results...') From a6c456e5172d5da25bd5a27ffe2baa6bd5d726f7 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Sat, 22 Jul 2017 15:22:33 -0400 Subject: [PATCH 076/519] pep fix --- packages/pandas-gbq/scripts/merge-py.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/pandas-gbq/scripts/merge-py.py b/packages/pandas-gbq/scripts/merge-py.py index 3596308e0a37..16e5f822ab40 100755 --- a/packages/pandas-gbq/scripts/merge-py.py +++ b/packages/pandas-gbq/scripts/merge-py.py @@ -233,6 +233,7 @@ def fix_version_from_branch(branch, versions): branch_ver = branch.replace("branch-", "") return filter(lambda x: x.name.startswith(branch_ver), versions)[-1] + pr_num = input("Which pull request would you like to merge? (e.g. 34): ") pr = get_json("%s/pulls/%s" % (GITHUB_API_BASE, pr_num)) From 5561eceeb4cd201cf4220938c0c313d8478af97c Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Sat, 22 Jul 2017 15:24:30 -0400 Subject: [PATCH 077/519] fix up whatsnew docs a bit --- packages/pandas-gbq/docs/source/changelog.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index b689ceee6099..abfe1a70cfc9 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -6,9 +6,9 @@ Changelog - Drop support for Python 3.4 (:issue:`40`) - The dataframe passed to ```.to_gbq(...., if_exists='append')``` needs to contain only a subset of the fields in the BigQuery schema. (:issue:`24`) -- Use the `google-auth `__ library for authentication because oauth2client is deprecated. (:issue:`39`) -- ``read_gbq`` now has a ``auth_local_webserver`` boolean argument for controlling whether to use web server or console flow when getting user credentials. Replaces `--noauth_local_webserver` command line argument. (:issue:`35`) -- ``read_gbq`` now displays the BigQuery Job ID and standard price in verbose output. (:issue:`70` and :issue:`71`) +- Use the `google-auth `__ library for authentication because ``oauth2client`` is deprecated. (:issue:`39`) +- :func:`read_gbq` now has a ``auth_local_webserver`` boolean argument for controlling whether to use web server or console flow when getting user credentials. Replaces `--noauth_local_webserver` command line argument. (:issue:`35`) +- `:func:`read_gbq` now displays the BigQuery Job ID and standard price in verbose output. (:issue:`70` and :issue:`71`) 0.1.6 / 2017-05-03 ------------------ @@ -18,12 +18,12 @@ Changelog 0.1.4 / 2017-03-17 ------------------ -- ``InvalidIndexColumn`` will be raised instead of ``InvalidColumnOrder`` in ``read_gbq`` when the index column specified does not exist in the BigQuery schema. (:issue:`6`) +- ``InvalidIndexColumn`` will be raised instead of ``InvalidColumnOrder`` in :func:`read_gbq` when the index column specified does not exist in the BigQuery schema. (:issue:`6`) 0.1.3 / 2017-03-04 ------------------ -- Bug with appending to a BigQuery table where fields have modes (NULLABLE,REQUIRED,REPEATED) specified. These modes were compared versus the remote schema and writing a table via ``to_gbq`` would previously raise. (:issue:`13`) +- Bug with appending to a BigQuery table where fields have modes (NULLABLE,REQUIRED,REPEATED) specified. These modes were compared versus the remote schema and writing a table via :func:`to_gbq` would previously raise. (:issue:`13`) 0.1.2 / 2017-02-23 ------------------ @@ -32,5 +32,5 @@ Initial release of transfered code from `pandas `__ -- ``read_gbq`` now stores ``INTEGER`` columns as ``dtype=object`` if they contain ``NULL`` values. Otherwise they are stored as ``int64``. This prevents precision lost for integers greather than 2**53. Furthermore ``FLOAT`` columns with values above 10**4 are no longer casted to ``int64`` which also caused precision loss `pandas-GH#14064 `__, and `pandas-GH#14305 `__ +- :func:`read_gbq` now allows query configuration preferences `pandas-GH#14742 `__ +- :func:`read_gbq` now stores ``INTEGER`` columns as ``dtype=object`` if they contain ``NULL`` values. Otherwise they are stored as ``int64``. This prevents precision lost for integers greather than 2**53. Furthermore ``FLOAT`` columns with values above 10**4 are no longer casted to ``int64`` which also caused precision loss `pandas-GH#14064 `__, and `pandas-GH#14305 `__ From c0feb66e54d256a7aa40e9f5649e089cecd727e8 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 24 Jul 2017 00:13:31 -0400 Subject: [PATCH 078/519] RLS 0.2.0 --- packages/pandas-gbq/docs/source/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index abfe1a70cfc9..efd25bcbbc48 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,7 +1,7 @@ Changelog ========= -0.2.0 / 2017-??-?? +0.2.0 / 2017-07-24 ------------------ - Drop support for Python 3.4 (:issue:`40`) From 13459e15f0a828a8c59817f88ee60179af5cdd8e Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 24 Jul 2017 23:30:57 -0400 Subject: [PATCH 079/519] DOC: Fix formatting --- packages/pandas-gbq/docs/source/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index efd25bcbbc48..d71f62877190 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -8,7 +8,7 @@ Changelog - The dataframe passed to ```.to_gbq(...., if_exists='append')``` needs to contain only a subset of the fields in the BigQuery schema. (:issue:`24`) - Use the `google-auth `__ library for authentication because ``oauth2client`` is deprecated. (:issue:`39`) - :func:`read_gbq` now has a ``auth_local_webserver`` boolean argument for controlling whether to use web server or console flow when getting user credentials. Replaces `--noauth_local_webserver` command line argument. (:issue:`35`) -- `:func:`read_gbq` now displays the BigQuery Job ID and standard price in verbose output. (:issue:`70` and :issue:`71`) +- :func:`read_gbq` now displays the BigQuery Job ID and standard price in verbose output. (:issue:`70` and :issue:`71`) 0.1.6 / 2017-05-03 ------------------ From d51a7a2c889d3ede6d830355219fd22cf12d6a3d Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 3 Aug 2017 11:23:28 -0400 Subject: [PATCH 080/519] next release in changelog.rst --- packages/pandas-gbq/docs/source/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index d71f62877190..b9e83f301ede 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,6 +1,9 @@ Changelog ========= +0.2.1 / 2017-??-?? +------------------ + 0.2.0 / 2017-07-24 ------------------ From f19486a001cbbe755e94ca34d00b98c01352c363 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 3 Aug 2017 11:42:31 -0400 Subject: [PATCH 081/519] TST: xfail delete/create table with different schema test (#77) --- packages/pandas-gbq/pandas_gbq/tests/test_gbq.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py index e8eda1d3f01a..930a9296196f 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -1033,6 +1033,17 @@ def test_upload_subset_columns_if_table_exists_append(self): private_key=_get_private_key_path()) assert result['num_rows'][0] == test_size * 2 + # This test is currently failing intermittently due to changes in the + # BigQuery backend. You can track the issue in the Google BigQuery issue + # tracker `here `__. + # Currently you need to stream data twice in order to successfully stream + # data when you delete and re-create a table with a different schema. + # Something to consider is that google-cloud-bigquery returns an array of + # streaming insert errors rather than raising an exception. In this + # scenario, a decision could be made by the user to check for streaming + # errors and retry as needed. See `Issue 75 + # `__ + @pytest.mark.xfail(reason="Delete/create table w/ different schema issue") def test_upload_data_if_table_exists_replace(self): test_id = "4" test_size = 10 From 2851c8928e272f6c46acab632bbd75583488c971 Mon Sep 17 00:00:00 2001 From: Alan Yee Date: Thu, 3 Aug 2017 10:50:40 -0700 Subject: [PATCH 082/519] DOC: Update intro.rst (#79) * Update intro.rst -Fix some grammar mistakes -Made explicit HTTPS calls * Update intro.rst Minor grammar fixes * Update intro.rst Correcting information via reviewer's recommendation * Update README.rst Remove unnecessary words --- packages/pandas-gbq/docs/source/intro.rst | 27 +++++++++++------------ 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/packages/pandas-gbq/docs/source/intro.rst b/packages/pandas-gbq/docs/source/intro.rst index 0b40be0567e5..fd64a4fb30a3 100644 --- a/packages/pandas-gbq/docs/source/intro.rst +++ b/packages/pandas-gbq/docs/source/intro.rst @@ -20,7 +20,7 @@ respectively. This is opposite to default pandas behaviour which will promote integer type to float in order to store NAs. -`See here for how this works in pandas `__ +`See here for how this works in pandas `__ While this trade-off works well for most cases, it breaks down for storing values greater than 2**53. Such values in BigQuery can represent identifiers @@ -31,26 +31,25 @@ and unnoticed precision lost for identifier is what we want to avoid. Authentication '''''''''''''' -Authentication to the Google ``BigQuery`` service is via ``OAuth 2.0``. -Is possible to authenticate with either user account credentials or service account credentials. +Authentication to the Google ``BigQuery`` service via ``OAuth 2.0`` +is possible with either user or service account credentials. -Authenticating with user account credentials is as simple as following the prompts in a browser window -which will be automatically opened for you. You will be authenticated to the specified -``BigQuery`` account using the product name ``pandas GBQ``. It is only possible on local host. -The remote authentication using user account credentials is not currently supported in pandas. +Authentication via user account credentials is as simple as following the prompts in a browser window +which will automatically open for you. You authenticate to the specified +``BigQuery`` account using the product name ``pandas GBQ``. +The remote authentication is supported via specifying ``auth_local_webserver`` in ``read_gbq``. Additional information on the authentication mechanism can be found `here `__. -Authentication with service account credentials is possible via the `'private_key'` parameter. This method -is particularly useful when working on remote servers (eg. jupyter iPython notebook on remote host). +Authentication via service account credentials is possible through the `'private_key'` parameter. This method +is particularly useful when working on remote servers (eg. Jupyter Notebooks on remote host). Additional information on service accounts can be found `here `__. -Authentication via ``application default credentials`` is also possible. This is only valid -if the parameter ``private_key`` is not provided. This method also requires that -the credentials can be fetched from the environment the code is running in. -Otherwise, the OAuth2 client-side authentication is used. -Additional information on +Authentication via ``application default credentials`` is also possible, but only valid +if the parameter ``private_key`` is not provided. This method requires that the +credentials can be fetched from the development environment. Otherwise, the OAuth2 +client-side authentication is used. Additional information can be found on `application default credentials `__. .. note:: From 0598f755acc457f38f7810d2a39512ff11b3afa6 Mon Sep 17 00:00:00 2001 From: Takashi Nishibayashi Date: Fri, 4 Aug 2017 21:41:44 +0900 Subject: [PATCH 083/519] ENH: Add timeout support (#76) * Add timeout support * Fix a pep8 issue * Check before fetch results * Add test * timeoutMs -> timeout_ms * Add changelog --- packages/pandas-gbq/docs/source/changelog.rst | 2 ++ packages/pandas-gbq/pandas_gbq/gbq.py | 12 ++++++++++++ packages/pandas-gbq/pandas_gbq/tests/test_gbq.py | 13 +++++++++++++ 3 files changed, 27 insertions(+) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index b9e83f301ede..26b6340b22ba 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -4,6 +4,8 @@ Changelog 0.2.1 / 2017-??-?? ------------------ +- :func:`read_gbq` now handles query configuration `query.timeoutMs` and stop waiting. (:issue:`76`) + 0.2.0 / 2017-07-24 ------------------ diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 452de5ef2331..9d1e5cacf0c1 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -172,6 +172,13 @@ class NotFoundException(ValueError): pass +class QueryTimeout(ValueError): + """ + Raised when the query job timeout + """ + pass + + class StreamingInsertError(ValueError): """ Raised when BigQuery reports a streaming insert error. @@ -536,6 +543,11 @@ def run_query(self, query, **kwargs): while not query_reply.get('jobComplete', False): self.print_elapsed_seconds(' Elapsed', 's. Waiting...') + + timeout_ms = job_config['query'].get('timeoutMs') + if timeout_ms and timeout_ms < self.get_elapsed_seconds() * 1000: + raise QueryTimeout('Query timeout: {} ms'.format(timeout_ms)) + try: query_reply = job_collection.getQueryResults( projectId=job_reference['projectId'], diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py index 930a9296196f..c9ac31dd30dc 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -884,6 +884,19 @@ def test_configuration_raises_value_error_with_multiple_config(self): private_key=_get_private_key_path(), configuration=config) + def test_timeout_configuration(self): + sql_statement = 'SELECT 1' + config = { + 'query': { + "timeoutMs": 1 + } + } + # Test that QueryTimeout error raises + with pytest.raises(gbq.QueryTimeout): + gbq.read_gbq(sql_statement, project_id=_get_project_id(), + private_key=_get_private_key_path(), + configuration=config) + def test_query_response_bytes(self): assert self.gbq_connector.sizeof_fmt(999) == "999.0 B" assert self.gbq_connector.sizeof_fmt(1024) == "1.0 KB" From 5b404fa88ec0048553ed90f3007b74833e3f77a1 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 4 Aug 2017 09:07:44 -0400 Subject: [PATCH 084/519] DOC: Update change log (#80) --- packages/pandas-gbq/docs/source/changelog.rst | 2 +- packages/pandas-gbq/pandas_gbq/gbq.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 26b6340b22ba..0f1d6e1bedac 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -4,7 +4,7 @@ Changelog 0.2.1 / 2017-??-?? ------------------ -- :func:`read_gbq` now handles query configuration `query.timeoutMs` and stop waiting. (:issue:`76`) +- :func:`read_gbq` now raises ``QueryTimeout`` if the request exceeds the ``query.timeoutMs`` value specified in the BigQuery configuration. (:issue:`76`) 0.2.0 / 2017-07-24 ------------------ diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 9d1e5cacf0c1..0c5386623f39 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -174,7 +174,8 @@ class NotFoundException(ValueError): class QueryTimeout(ValueError): """ - Raised when the query job timeout + Raised when the query request exceeds the timeoutMs value specified in the + BigQuery configuration. """ pass From 9d8eee65b8b9f3d4f6814c8877ecf28e3bf9fbfc Mon Sep 17 00:00:00 2001 From: Jonathan Prates Date: Wed, 13 Sep 2017 15:31:21 -0300 Subject: [PATCH 085/519] ENH: Add support for env variable PANDAS_GBQ_CREDENTIALS_FILE (#87) --- packages/pandas-gbq/docs/source/changelog.rst | 2 ++ packages/pandas-gbq/pandas_gbq/gbq.py | 15 ++++++++++++--- packages/pandas-gbq/pandas_gbq/tests/test_gbq.py | 8 ++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 0f1d6e1bedac..fb841b34ccd2 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -5,6 +5,8 @@ Changelog ------------------ - :func:`read_gbq` now raises ``QueryTimeout`` if the request exceeds the ``query.timeoutMs`` value specified in the BigQuery configuration. (:issue:`76`) +- Environment variable ``PANDAS_GBQ_CREDENTIALS_FILE`` can now be used to override the default location where the BigQuery user account credentials are stored. (:issue:`86`) + 0.2.0 / 2017-07-24 ------------------ diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 0c5386623f39..d871c0680578 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -5,6 +5,7 @@ import uuid import time import sys +import os import numpy as np @@ -279,7 +280,7 @@ def load_user_account_credentials(self): from google.oauth2.credentials import Credentials try: - with open('bigquery_credentials.dat') as credentials_file: + with open(_get_credentials_file()) as credentials_file: credentials_json = json.load(credentials_file) except (IOError, ValueError): return None @@ -307,7 +308,7 @@ def save_user_account_credentials(self, credentials): .. versionadded 0.2.0 """ try: - with open('bigquery_credentials.dat', 'w') as credentials_file: + with open(_get_credentials_file(), 'w') as credentials_file: credentials_json = { 'refresh_token': credentials.refresh_token, 'id_token': credentials.id_token, @@ -790,6 +791,11 @@ def delete_and_recreate_table(self, dataset_id, table_id, table_schema): sleep(delay) +def _get_credentials_file(): + return os.environ.get( + 'PANDAS_GBQ_CREDENTIALS_FILE', 'bigquery_credentials.dat') + + def _parse_data(schema, rows): # see: # http://pandas.pydata.org/pandas-docs/dev/missing_data.html @@ -875,7 +881,10 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, authentication (eg. jupyter iPython notebook on remote host) auth_local_webserver : boolean, default False Use the [local webserver flow] instead of the [console flow] when - getting user credentials. + getting user credentials. A file named bigquery_credentials.dat will + be created in current dir. You can also set PANDAS_GBQ_CREDENTIALS_FILE + environment variable so as to define a specific path to store this + credential (eg. /etc/keys/bigquery.dat). .. [local webserver flow] http://google-auth-oauthlib.readthedocs.io/en/latest/reference/google_auth_oauthlib.flow.html#google_auth_oauthlib.flow.InstalledAppFlow.run_local_server diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py index c9ac31dd30dc..f61e4b52134e 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -321,6 +321,14 @@ def test_import_google_api_python_client(self): from googleapiclient.discovery import build # noqa from googleapiclient.errors import HttpError # noqa + def test_should_return_credentials_path_set_by_env_var(self): + import mock + env = {'PANDAS_GBQ_CREDENTIALS_FILE': '/tmp/dummy.dat'} + with mock.patch.dict('os.environ', env): + assert gbq._get_credentials_file() == '/tmp/dummy.dat' + + assert gbq._get_credentials_file() == 'bigquery_credentials.dat' + def test_should_return_bigquery_integers_as_python_ints(self): result = gbq._parse_entry(1, 'INTEGER') assert result == int(1) From 0156974c8794d451a90a72610ab2480c6d39aedd Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 19 Sep 2017 11:54:37 -0400 Subject: [PATCH 086/519] ENH: Save BigQuery account credentials in a hidden user folder (#83) * ENH: Save BigQuery account credentials in a hidden user folder instead of cwd * Revert version bump --- packages/pandas-gbq/docs/source/changelog.rst | 2 +- packages/pandas-gbq/docs/source/intro.rst | 4 +- packages/pandas-gbq/pandas_gbq/gbq.py | 47 +++++++++++++++++-- .../pandas-gbq/pandas_gbq/tests/test_gbq.py | 1 + 4 files changed, 49 insertions(+), 5 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index fb841b34ccd2..dc35067e43da 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -6,7 +6,7 @@ Changelog - :func:`read_gbq` now raises ``QueryTimeout`` if the request exceeds the ``query.timeoutMs`` value specified in the BigQuery configuration. (:issue:`76`) - Environment variable ``PANDAS_GBQ_CREDENTIALS_FILE`` can now be used to override the default location where the BigQuery user account credentials are stored. (:issue:`86`) - +- BigQuery user account credentials are now stored in an application-specific hidden user folder on the operating system. (:issue:`41`) 0.2.0 / 2017-07-24 ------------------ diff --git a/packages/pandas-gbq/docs/source/intro.rst b/packages/pandas-gbq/docs/source/intro.rst index fd64a4fb30a3..3cac399a6ddc 100644 --- a/packages/pandas-gbq/docs/source/intro.rst +++ b/packages/pandas-gbq/docs/source/intro.rst @@ -37,7 +37,9 @@ is possible with either user or service account credentials. Authentication via user account credentials is as simple as following the prompts in a browser window which will automatically open for you. You authenticate to the specified ``BigQuery`` account using the product name ``pandas GBQ``. -The remote authentication is supported via specifying ``auth_local_webserver`` in ``read_gbq``. +The remote authentication is supported via the ``auth_local_webserver`` in ``read_gbq``. By default, +account credentials are stored in an application-specific hidden user folder on the operating system. You +can override the default credentials location via the ``PANDAS_GBQ_CREDENTIALS_FILE`` environment variable. Additional information on the authentication mechanism can be found `here `__. diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index d871c0680578..9473b0822908 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -208,6 +208,7 @@ def __init__(self, project_id, reauth=False, verbose=False, self.private_key = private_key self.auth_local_webserver = auth_local_webserver self.dialect = dialect + self.credentials_path = _get_credentials_file() self.credentials = self.get_credentials() self.service = self.get_service() @@ -279,8 +280,21 @@ def load_user_account_credentials(self): from google_auth_httplib2 import Request from google.oauth2.credentials import Credentials + # Use the default credentials location under ~/.config and the + # equivalent directory on windows if the user has not specified a + # credentials path. + if not self.credentials_path: + self.credentials_path = self.get_default_credentials_path() + + # Previously, pandas-gbq saved user account credentials in the + # current working directory. If the bigquery_credentials.dat file + # exists in the current working directory, move the credentials to + # the new default location. + if os.path.isfile('bigquery_credentials.dat'): + os.rename('bigquery_credentials.dat', self.credentials_path) + try: - with open(_get_credentials_file()) as credentials_file: + with open(self.credentials_path) as credentials_file: credentials_json = json.load(credentials_file) except (IOError, ValueError): return None @@ -301,6 +315,33 @@ def load_user_account_credentials(self): return _try_credentials(self.project_id, credentials) + def get_default_credentials_path(self): + """ + Gets the default path to the BigQuery credentials + + .. versionadded 0.3.0 + + Returns + ------- + Path to the BigQuery credentials + """ + + import os + + if os.name == 'nt': + config_path = os.environ['APPDATA'] + else: + config_path = os.path.join(os.path.expanduser('~'), '.config') + + config_path = os.path.join(config_path, 'pandas_gbq') + + # Create a pandas_gbq directory in an application-specific hidden + # user folder on the operating system. + if not os.path.exists(config_path): + os.makedirs(config_path) + + return os.path.join(config_path, 'bigquery_credentials.dat') + def save_user_account_credentials(self, credentials): """ Saves user account credentials to a local file. @@ -308,7 +349,7 @@ def save_user_account_credentials(self, credentials): .. versionadded 0.2.0 """ try: - with open(_get_credentials_file(), 'w') as credentials_file: + with open(self.credentials_path, 'w') as credentials_file: credentials_json = { 'refresh_token': credentials.refresh_token, 'id_token': credentials.id_token, @@ -793,7 +834,7 @@ def delete_and_recreate_table(self, dataset_id, table_id, table_schema): def _get_credentials_file(): return os.environ.get( - 'PANDAS_GBQ_CREDENTIALS_FILE', 'bigquery_credentials.dat') + 'PANDAS_GBQ_CREDENTIALS_FILE') def _parse_data(schema, rows): diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py index f61e4b52134e..62b72dbc884f 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -1388,6 +1388,7 @@ def setup_method(self, method): # put here any instruction you want to be run *BEFORE* *EVERY* test # is executed. + gbq.GbqConnector(_get_project_id(), auth_local_webserver=True) self.dataset_prefix = _get_dataset_prefix_random() clean_gbq_environment(self.dataset_prefix) self.destination_table = "{0}{1}.{2}".format(self.dataset_prefix, "2", From 0ab90b4b71017f8c5a55d8f5dc484c50df84d21c Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 27 Nov 2017 14:59:50 -0800 Subject: [PATCH 087/519] Prepare for 0.2.1 release. (#94) * Prepare for 0.2.1 release. * Catch ImportError instead of bare except. Fixes lint errors in CI builds. --- packages/pandas-gbq/docs/source/changelog.rst | 2 +- packages/pandas-gbq/pandas_gbq/gbq.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index dc35067e43da..e9998a598d20 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,7 +1,7 @@ Changelog ========= -0.2.1 / 2017-??-?? +0.2.1 / 2017-11-27 ------------------ - :func:`read_gbq` now raises ``QueryTimeout`` if the request exceeds the ``query.timeoutMs`` value specified in the BigQuery configuration. (:issue:`76`) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 9473b0822908..da7dd21dc59a 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -530,7 +530,7 @@ def process_insert_errors(self, insert_errors): def run_query(self, query, **kwargs): try: from googleapiclient.errors import HttpError - except: + except ImportError: from apiclient.errors import HttpError from google.auth.exceptions import RefreshError @@ -663,7 +663,7 @@ def run_query(self, query, **kwargs): def load_data(self, dataframe, dataset_id, table_id, chunksize): try: from googleapiclient.errors import HttpError - except: + except ImportError: from apiclient.errors import HttpError job_id = uuid.uuid4().hex @@ -737,7 +737,7 @@ def schema(self, dataset_id, table_id): try: from googleapiclient.errors import HttpError - except: + except ImportError: from apiclient.errors import HttpError try: @@ -1162,7 +1162,7 @@ def __init__(self, project_id, dataset_id, reauth=False, verbose=False, private_key=None): try: from googleapiclient.errors import HttpError - except: + except ImportError: from apiclient.errors import HttpError self.http_error = HttpError self.dataset_id = dataset_id @@ -1261,7 +1261,7 @@ def __init__(self, project_id, reauth=False, verbose=False, private_key=None): try: from googleapiclient.errors import HttpError - except: + except ImportError: from apiclient.errors import HttpError self.http_error = HttpError super(_Dataset, self).__init__(project_id, reauth, verbose, From 2a5d7bf99f6649e1a96e72964bb5d0f4fcd956c0 Mon Sep 17 00:00:00 2001 From: "Jason Q. Ng" Date: Wed, 20 Dec 2017 17:17:10 -0500 Subject: [PATCH 088/519] ENH: Convert calls to BigQuery API to use google-cloud-python (#25) * Moved read_query to GbqConnector, update credentials and client generation to GbqConnector, and remove wait_for_job * Move sizeof_fmt back into GbqConnector * Convert rest of methods to use google-cloud-bigquery - Removes references to google-api-client-library and httplib2. - Updates PR to not make any surface-level changes to the API, only swaps out the dependencies. - Updates PR to use latest version of google-cloud-bigquery. * Ignore mode property when comparing schemas. * Document new dependency on google-cloud-bigquery. * Document dependencies for previous verions. Also says which libraries are no longer required, for easier upgrades. * Add deps and StreamingInsertError to changelog. --- .../pandas-gbq/ci/requirements-2.7-0.19.2.pip | 4 +- .../pandas-gbq/ci/requirements-3.5-0.18.1.pip | 3 +- .../ci/requirements-3.6-0.20.1.conda | 3 +- .../pandas-gbq/ci/requirements-3.6-MASTER.pip | 3 +- packages/pandas-gbq/docs/source/changelog.rst | 7 + packages/pandas-gbq/docs/source/install.rst | 15 +- packages/pandas-gbq/pandas_gbq/gbq.py | 557 ++++++------------ .../pandas-gbq/pandas_gbq/tests/test_gbq.py | 42 +- packages/pandas-gbq/requirements.txt | 4 +- packages/pandas-gbq/setup.py | 4 +- 10 files changed, 231 insertions(+), 411 deletions(-) diff --git a/packages/pandas-gbq/ci/requirements-2.7-0.19.2.pip b/packages/pandas-gbq/ci/requirements-2.7-0.19.2.pip index 852dc1536035..cd94478a457c 100644 --- a/packages/pandas-gbq/ci/requirements-2.7-0.19.2.pip +++ b/packages/pandas-gbq/ci/requirements-2.7-0.19.2.pip @@ -1,7 +1,5 @@ -google-api-python-client google-auth -google-auth-httplib2 google-auth-oauthlib PyCrypto -python-gflags mock +google-cloud-bigquery diff --git a/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip b/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip index 6fb8a03d81fc..18369345e8a4 100644 --- a/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip +++ b/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip @@ -1,5 +1,4 @@ -google-api-python-client==1.6.0 google-auth==1.0.0 -google-auth-httplib2==0.0.1 google-auth-oauthlib==0.0.1 mock +google-cloud-bigquery==0.28.0 diff --git a/packages/pandas-gbq/ci/requirements-3.6-0.20.1.conda b/packages/pandas-gbq/ci/requirements-3.6-0.20.1.conda index a1608720f3c6..b52f2aeb3289 100644 --- a/packages/pandas-gbq/ci/requirements-3.6-0.20.1.conda +++ b/packages/pandas-gbq/ci/requirements-3.6-0.20.1.conda @@ -1,5 +1,4 @@ -google-api-python-client google-auth -google-auth-httplib2 google-auth-oauthlib mock +google-cloud-bigquery diff --git a/packages/pandas-gbq/ci/requirements-3.6-MASTER.pip b/packages/pandas-gbq/ci/requirements-3.6-MASTER.pip index a1608720f3c6..b52f2aeb3289 100644 --- a/packages/pandas-gbq/ci/requirements-3.6-MASTER.pip +++ b/packages/pandas-gbq/ci/requirements-3.6-MASTER.pip @@ -1,5 +1,4 @@ -google-api-python-client google-auth -google-auth-httplib2 google-auth-oauthlib mock +google-cloud-bigquery diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index e9998a598d20..b6684582acae 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,6 +1,13 @@ Changelog ========= +0.3.0 / 2017-??-?? +------------------ + +- Use the `google-cloud-bigquery `__ library for API calls. The ``google-cloud-bigquery`` package is a new dependency, and dependencies on ``google-api-python-client`` and ``httplib2`` are removed. See the `installation guide `__ for more details. (:issue:`93`) +- :func:`to_gbq` now uses a load job instead of the streaming API. (:issue:`75`) +- Remove ``StreamingInsertError`` class, as it is no longer used by :func:`to_gbq`. (:issue:`75`) + 0.2.1 / 2017-11-27 ------------------ diff --git a/packages/pandas-gbq/docs/source/install.rst b/packages/pandas-gbq/docs/source/install.rst index 2b701fd2443f..c64c7939d24f 100644 --- a/packages/pandas-gbq/docs/source/install.rst +++ b/packages/pandas-gbq/docs/source/install.rst @@ -37,8 +37,17 @@ Dependencies This module requires following additional dependencies: -- `httplib2 `__: HTTP client -- `google-api-python-client `__: Google's API client - `google-auth `__: authentication and authorization for Google's API - `google-auth-oauthlib `__: integration with `oauthlib `__ for end-user authentication -- `google-auth-httplib2 `__: adapter to use ``httplib2`` HTTP client with ``google-auth`` +- `google-cloud-bigquery `__: Google Cloud client library for BigQuery + +.. note:: + + The dependency on `google-cloud-bigquery `__ is new in version 0.3.0 of ``pandas-gbq``. + Versions less than 0.3.0 required the following dependencies: + + - `httplib2 `__: HTTP client (no longer required) + - `google-api-python-client `__: Google's API client (no longer required, replaced by `google-cloud-bigquery `__:) + - `google-auth `__: authentication and authorization for Google's API + - `google-auth-oauthlib `__: integration with `oauthlib `__ for end-user authentication + - `google-auth-httplib2 `__: adapter to use ``httplib2`` HTTP client with ``google-auth`` (no longer required) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index da7dd21dc59a..46a246e528e6 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -1,17 +1,16 @@ import warnings from datetime import datetime import json -from time import sleep -import uuid import time +from time import sleep import sys import os import numpy as np from distutils.version import StrictVersion -from pandas import compat, DataFrame, concat -from pandas.compat import lzip, bytes_to_str +from pandas import compat, DataFrame +from pandas.compat import lzip def _check_google_client_version(): @@ -22,31 +21,24 @@ def _check_google_client_version(): except ImportError: raise ImportError('Could not import pkg_resources (setuptools).') - # Version 1.6.0 is the first version to support google-auth. - # https://github.com/google/google-api-python-client/blob/master/CHANGELOG - google_api_minimum_version = '1.6.0' + # Version 0.28.0 includes many changes compared to previous versions + # https://github.com/GoogleCloudPlatform/google-cloud-python/blob/master/bigquery/CHANGELOG.md + bigquery_client_minimum_version = '0.28.0' - _GOOGLE_API_CLIENT_VERSION = pkg_resources.get_distribution( - 'google-api-python-client').version + _BIGQUERY_CLIENT_VERSION = pkg_resources.get_distribution( + 'google-cloud-bigquery').version - if (StrictVersion(_GOOGLE_API_CLIENT_VERSION) < - StrictVersion(google_api_minimum_version)): - raise ImportError('pandas requires google-api-python-client >= {0} ' + if (StrictVersion(_BIGQUERY_CLIENT_VERSION) < + StrictVersion(bigquery_client_minimum_version)): + raise ImportError('pandas requires google-cloud-bigquery >= {0} ' 'for Google BigQuery support, ' 'current version {1}' - .format(google_api_minimum_version, - _GOOGLE_API_CLIENT_VERSION)) + .format(bigquery_client_minimum_version, + _BIGQUERY_CLIENT_VERSION)) def _test_google_api_imports(): - try: - import httplib2 # noqa - except ImportError as ex: - raise ImportError( - 'pandas requires httplib2 for Google BigQuery support: ' - '{0}'.format(ex)) - try: from google_auth_oauthlib.flow import InstalledAppFlow # noqa except ImportError as ex: @@ -55,50 +47,35 @@ def _test_google_api_imports(): 'support: {0}'.format(ex)) try: - from google_auth_httplib2 import AuthorizedHttp # noqa - from google_auth_httplib2 import Request # noqa - except ImportError as ex: - raise ImportError( - 'pandas requires google-auth-httplib2 for Google BigQuery ' - 'support: {0}'.format(ex)) - - try: - from googleapiclient.discovery import build # noqa - from googleapiclient.errors import HttpError # noqa + import google.auth # noqa except ImportError as ex: raise ImportError( - "pandas requires google-api-python-client for Google BigQuery " - "support: {0}".format(ex)) + "pandas requires google-auth for Google BigQuery support: " + "{0}".format(ex)) try: - import google.auth # noqa + from google.cloud import bigquery # noqa except ImportError as ex: raise ImportError( - "pandas requires google-auth for Google BigQuery support: " + "pandas requires google-cloud-python for Google BigQuery support: " "{0}".format(ex)) _check_google_client_version() def _try_credentials(project_id, credentials): - import httplib2 - from googleapiclient.discovery import build - import googleapiclient.errors - from google_auth_httplib2 import AuthorizedHttp + from google.cloud import bigquery + import google.api_core.exceptions if credentials is None: return None - http = httplib2.Http() try: - authed_http = AuthorizedHttp(credentials, http=http) - bigquery_service = build('bigquery', 'v2', http=authed_http) + client = bigquery.Client(project=project_id, credentials=credentials) # Check if the application has rights to the BigQuery project - jobs = bigquery_service.jobs() - job_data = {'configuration': {'query': {'query': 'SELECT 1'}}} - jobs.insert(projectId=project_id, body=job_data).execute() + client.query('SELECT 1').result() return credentials - except googleapiclient.errors.Error: + except google.api_core.exceptions.GoogleAPIError: return None @@ -181,14 +158,6 @@ class QueryTimeout(ValueError): pass -class StreamingInsertError(ValueError): - """ - Raised when BigQuery reports a streaming insert error. - For more information see `Streaming Data Into BigQuery - `__ - """ - - class TableCreationError(ValueError): """ Raised when the create table method fails @@ -202,6 +171,9 @@ class GbqConnector(object): def __init__(self, project_id, reauth=False, verbose=False, private_key=None, auth_local_webserver=False, dialect='legacy'): + from google.api_core.exceptions import GoogleAPIError + from google.api_core.exceptions import ClientError + self.http_error = (ClientError, GoogleAPIError) self.project_id = project_id self.reauth = reauth self.verbose = verbose @@ -210,7 +182,7 @@ def __init__(self, project_id, reauth=False, verbose=False, self.dialect = dialect self.credentials_path = _get_credentials_file() self.credentials = self.get_credentials() - self.service = self.get_service() + self.client = self.get_client() # BQ Queries costs $5 per TB. First 1 TB per month is free # see here for more: https://cloud.google.com/bigquery/pricing @@ -276,8 +248,7 @@ def load_user_account_credentials(self): credentials do not have access to the project (self.project_id) on BigQuery. """ - import httplib2 - from google_auth_httplib2 import Request + import google.auth.transport.requests from google.oauth2.credentials import Credentials # Use the default credentials location under ~/.config and the @@ -309,8 +280,7 @@ def load_user_account_credentials(self): scopes=credentials_json.get('scopes')) # Refresh the token before trying to use it. - http = httplib2.Http() - request = Request(http) + request = google.auth.transport.requests.Request() credentials.refresh(request) return _try_credentials(self.project_id, credentials) @@ -411,8 +381,7 @@ def get_user_account_credentials(self): return credentials def get_service_account_credentials(self): - import httplib2 - from google_auth_httplib2 import Request + import google.auth.transport.requests from google.oauth2.service_account import Credentials from os.path import isfile @@ -435,8 +404,7 @@ def get_service_account_credentials(self): credentials = credentials.with_scopes([self.scope]) # Refresh the token before trying to use it. - http = httplib2.Http() - request = Request(http) + request = google.auth.transport.requests.Request() credentials.refresh(request) return credentials @@ -475,70 +443,25 @@ def sizeof_fmt(num, suffix='B'): num /= 1024.0 return fmt % (num, 'Y', suffix) - def get_service(self): - import httplib2 - from google_auth_httplib2 import AuthorizedHttp - from googleapiclient.discovery import build - - http = httplib2.Http() - authed_http = AuthorizedHttp( - self.credentials, http=http) - bigquery_service = build('bigquery', 'v2', http=authed_http) - - return bigquery_service + def get_client(self): + from google.cloud import bigquery + return bigquery.Client( + project=self.project_id, credentials=self.credentials) @staticmethod def process_http_error(ex): # See `BigQuery Troubleshooting Errors # `__ - status = json.loads(bytes_to_str(ex.content))['error'] - errors = status.get('errors', None) - - if errors: - for error in errors: - reason = error['reason'] - message = error['message'] - - raise GenericGBQException( - "Reason: {0}, Message: {1}".format(reason, message)) - - raise GenericGBQException(errors) - - def process_insert_errors(self, insert_errors): - for insert_error in insert_errors: - row = insert_error['index'] - errors = insert_error.get('errors', None) - for error in errors: - reason = error['reason'] - message = error['message'] - location = error['location'] - error_message = ('Error at Row: {0}, Reason: {1}, ' - 'Location: {2}, Message: {3}' - .format(row, reason, location, message)) - - # Report all error messages if verbose is set - if self.verbose: - self._print(error_message) - else: - raise StreamingInsertError(error_message + - '\nEnable verbose logging to ' - 'see all errors') - - raise StreamingInsertError + raise GenericGBQException("Reason: {0}".format(ex)) def run_query(self, query, **kwargs): - try: - from googleapiclient.errors import HttpError - except ImportError: - from apiclient.errors import HttpError from google.auth.exceptions import RefreshError - - job_collection = self.service.jobs() + from google.cloud.bigquery import QueryJobConfig + from concurrent.futures import TimeoutError job_config = { 'query': { - 'query': query, 'useLegacySql': self.dialect == 'legacy' # 'allowLargeResults', 'createDisposition', # 'preserveNulls', destinationTable, useQueryCache @@ -550,24 +473,24 @@ def run_query(self, query, **kwargs): raise ValueError("Only one job type must be specified, but " "given {}".format(','.join(config.keys()))) if 'query' in config: - if 'query' in config['query'] and query is not None: - raise ValueError("Query statement can't be specified " - "inside config while it is specified " - "as parameter") + if 'query' in config['query']: + if query is not None: + raise ValueError("Query statement can't be specified " + "inside config while it is specified " + "as parameter") + query = config['query']['query'] + del config['query']['query'] job_config['query'].update(config['query']) else: raise ValueError("Only 'query' job type is supported") - job_data = { - 'configuration': job_config - } - self._start_timer() try: self._print('Requesting query... ', end="") - query_reply = job_collection.insert( - projectId=self.project_id, body=job_data).execute() + query_reply = self.client.query( + query, + job_config=QueryJobConfig.from_api_repr(job_config['query'])) self._print('ok.') except (RefreshError, ValueError): if self.private_key: @@ -577,96 +500,71 @@ def run_query(self, query, **kwargs): raise AccessDenied( "The credentials have been revoked or expired, " "please re-run the application to re-authorize") - except HttpError as ex: + except self.http_error as ex: self.process_http_error(ex) - job_reference = query_reply['jobReference'] - job_id = job_reference['jobId'] + job_id = query_reply.job_id self._print('Job ID: %s\nQuery running...' % job_id) - while not query_reply.get('jobComplete', False): + while query_reply.state != 'DONE': self.print_elapsed_seconds(' Elapsed', 's. Waiting...') timeout_ms = job_config['query'].get('timeoutMs') if timeout_ms and timeout_ms < self.get_elapsed_seconds() * 1000: raise QueryTimeout('Query timeout: {} ms'.format(timeout_ms)) + timeout_sec = 1.0 + if timeout_ms: + # Wait at most 1 second so we can show progress bar + timeout_sec = min(1.0, timeout_ms / 1000.0) + try: - query_reply = job_collection.getQueryResults( - projectId=job_reference['projectId'], - jobId=job_id).execute() - except HttpError as ex: + query_reply.result(timeout=timeout_sec) + except TimeoutError: + # Use our own timeout logic + pass + except self.http_error as ex: self.process_http_error(ex) if self.verbose: - if query_reply['cacheHit']: + if query_reply.cache_hit: self._print('Query done.\nCache hit.\n') else: - bytes_processed = int(query_reply.get( - 'totalBytesProcessed', '0')) - self._print('Query done.\nProcessed: {}'.format( - self.sizeof_fmt(bytes_processed))) + bytes_processed = query_reply.total_bytes_processed or 0 + bytes_billed = query_reply.total_bytes_billed or 0 + self._print('Query done.\nProcessed: {} Billed: {}'.format( + self.sizeof_fmt(bytes_processed), + self.sizeof_fmt(bytes_billed))) self._print('Standard price: ${:,.2f} USD\n'.format( - bytes_processed * self.query_price_for_TB)) + bytes_billed * self.query_price_for_TB)) self._print('Retrieving results...') - total_rows = int(query_reply['totalRows']) - result_pages = list() - seen_page_tokens = list() - current_row = 0 - # Only read schema on first page - schema = query_reply['schema'] - - # Loop through each page of data - while 'rows' in query_reply and current_row < total_rows: - page = query_reply['rows'] - result_pages.append(page) - current_row += len(page) - - self.print_elapsed_seconds( - ' Got page: {}; {}% done. Elapsed'.format( - len(result_pages), - round(100.0 * current_row / total_rows))) - - if current_row == total_rows: - break - - page_token = query_reply.get('pageToken', None) - - if not page_token and current_row < total_rows: - raise InvalidPageToken("Required pageToken was missing. " - "Received {0} of {1} rows" - .format(current_row, total_rows)) - - elif page_token in seen_page_tokens: - raise InvalidPageToken("A duplicate pageToken was returned") - - seen_page_tokens.append(page_token) - - try: - query_reply = job_collection.getQueryResults( - projectId=job_reference['projectId'], - jobId=job_id, - pageToken=page_token).execute() - except HttpError as ex: - self.process_http_error(ex) - - if current_row < total_rows: - raise InvalidPageToken() + try: + rows_iter = query_reply.result() + except self.http_error as ex: + self.process_http_error(ex) + result_rows = list(rows_iter) + total_rows = rows_iter.total_rows + schema = { + 'fields': [ + field.to_api_repr() + for field in rows_iter.schema], + } # print basic query stats self._print('Got {} rows.\n'.format(total_rows)) - return schema, result_pages + return schema, result_rows def load_data(self, dataframe, dataset_id, table_id, chunksize): - try: - from googleapiclient.errors import HttpError - except ImportError: - from apiclient.errors import HttpError + from google.cloud.bigquery import LoadJobConfig + from six import StringIO - job_id = uuid.uuid4().hex + destination_table = self.client.dataset(dataset_id).table(table_id) + job_config = LoadJobConfig() + job_config.write_disposition = 'WRITE_APPEND' + job_config.source_format = 'NEWLINE_DELIMITED_JSON' rows = [] remaining_rows = len(dataframe) @@ -674,44 +572,25 @@ def load_data(self, dataframe, dataset_id, table_id, chunksize): self._print("\n\n") for index, row in dataframe.reset_index(drop=True).iterrows(): - row_dict = dict() - row_dict['json'] = json.loads(row.to_json(force_ascii=False, - date_unit='s', - date_format='iso')) - row_dict['insertId'] = job_id + str(index) - rows.append(row_dict) + row_json = row.to_json( + force_ascii=False, date_unit='s', date_format='iso') + rows.append(row_json) remaining_rows -= 1 if (len(rows) % chunksize == 0) or (remaining_rows == 0): - self._print("\rStreaming Insert is {0}% Complete".format( + self._print("\rLoad is {0}% Complete".format( ((total_rows - remaining_rows) * 100) / total_rows)) - body = {'rows': rows} + body = StringIO('{}\n'.format('\n'.join(rows))) try: - response = self.service.tabledata().insertAll( - projectId=self.project_id, - datasetId=dataset_id, - tableId=table_id, - body=body).execute() - except HttpError as ex: + self.client.load_table_from_file( + body, + destination_table, + job_config=job_config).result() + except self.http_error as ex: self.process_http_error(ex) - # For streaming inserts, even if you receive a success HTTP - # response code, you'll need to check the insertErrors property - # of the response to determine if the row insertions were - # successful, because it's possible that BigQuery was only - # partially successful at inserting the rows. See the `Success - # HTTP Response Codes - # `__ - # section - - insert_errors = response.get('insertErrors', None) - if insert_errors: - self.process_insert_errors(insert_errors) - - sleep(1) # Maintains the inserts "per second" rate per API rows = [] self._print("\n") @@ -734,24 +613,20 @@ def schema(self, dataset_id, table_id): list of dicts Fields representing the schema """ + table_ref = self.client.dataset(dataset_id).table(table_id) try: - from googleapiclient.errors import HttpError - except ImportError: - from apiclient.errors import HttpError + table = self.client.get_table(table_ref) + remote_schema = table.schema - try: - remote_schema = self.service.tables().get( - projectId=self.project_id, - datasetId=dataset_id, - tableId=table_id).execute()['schema'] - - remote_fields = [{'name': field_remote['name'], - 'type': field_remote['type']} - for field_remote in remote_schema['fields']] + remote_fields = [ + field_remote.to_api_repr() for field_remote in remote_schema] + for field in remote_fields: + field['type'] = field['type'].upper() + field['mode'] = field['mode'].upper() return remote_fields - except HttpError as ex: + except self.http_error as ex: self.process_http_error(ex) def verify_schema(self, dataset_id, table_id, schema): @@ -781,6 +656,14 @@ def verify_schema(self, dataset_id, table_id, schema): key=lambda x: x['name']) fields_local = sorted(schema['fields'], key=lambda x: x['name']) + # Ignore mode when comparing schemas. + for field in fields_local: + if 'mode' in field: + del field['mode'] + for field in fields_remote: + if 'mode' in field: + del field['mode'] + return fields_remote == fields_local def schema_is_subset(self, dataset_id, table_id, schema): @@ -809,6 +692,14 @@ def schema_is_subset(self, dataset_id, table_id, schema): fields_remote = self.schema(dataset_id, table_id) fields_local = schema['fields'] + # Ignore mode when comparing schemas. + for field in fields_local: + if 'mode' in field: + del field['mode'] + for field in fields_remote: + if 'mode' in field: + del field['mode'] + return all(field in fields_remote for field in fields_local) def delete_and_recreate_table(self, dataset_id, table_id, table_schema): @@ -847,44 +738,30 @@ def _parse_data(schema, rows): fields = schema['fields'] col_types = [field['type'] for field in fields] col_names = [str(field['name']) for field in fields] - col_dtypes = [dtype_map.get(field['type'], object) for field in fields] + col_dtypes = [ + dtype_map.get(field['type'].upper(), object) + for field in fields + ] page_array = np.zeros((len(rows),), dtype=lzip(col_names, col_dtypes)) - for row_num, raw_row in enumerate(rows): - entries = raw_row.get('f', []) - for col_num, field_type in enumerate(col_types): - field_value = _parse_entry(entries[col_num].get('v', ''), - field_type) + for row_num, entries in enumerate(rows): + for col_num in range(len(col_types)): + field_value = entries[col_num] page_array[row_num][col_num] = field_value return DataFrame(page_array, columns=col_names) -def _parse_entry(field_value, field_type): - if field_value is None or field_value == 'null': - return None - if field_type == 'INTEGER': - return int(field_value) - elif field_type == 'FLOAT': - return float(field_value) - elif field_type == 'TIMESTAMP': - timestamp = datetime.utcfromtimestamp(float(field_value)) - return np.datetime64(timestamp) - elif field_type == 'BOOLEAN': - return field_value == 'true' - return field_value - - def read_gbq(query, project_id=None, index_col=None, col_order=None, reauth=False, verbose=True, private_key=None, auth_local_webserver=False, dialect='legacy', **kwargs): - r"""Load data from Google BigQuery. + r"""Load data from Google BigQuery using google-cloud-python The main method a user calls to execute a Query in Google BigQuery and read results into a pandas DataFrame. - Google BigQuery API Client Library v2 for Python is used. + The Google Cloud library is used. Documentation is available `here - `__ + `__ Authentication to the Google BigQuery service is via OAuth 2.0. @@ -967,16 +844,8 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, connector = GbqConnector( project_id, reauth=reauth, verbose=verbose, private_key=private_key, dialect=dialect, auth_local_webserver=auth_local_webserver) - schema, pages = connector.run_query(query, **kwargs) - dataframe_list = [] - while len(pages) > 0: - page = pages.pop() - dataframe_list.append(_parse_data(schema, page)) - - if len(dataframe_list) > 0: - final_df = concat(dataframe_list, ignore_index=True) - else: - final_df = _parse_data(schema, []) + schema, rows = connector.run_query(query, **kwargs) + final_df = _parse_data(schema, rows) # Reindex the DataFrame on the provided column if index_col is not None: @@ -1001,10 +870,10 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, # if they dont have any nulls type_map = {'BOOLEAN': bool, 'INTEGER': int} for field in schema['fields']: - if field['type'] in type_map and \ + if field['type'].upper() in type_map and \ final_df[field['name']].notnull().all(): final_df[field['name']] = \ - final_df[field['name']].astype(type_map[field['type']]) + final_df[field['name']].astype(type_map[field['type'].upper()]) connector.print_elapsed_seconds( 'Total time taken', @@ -1160,11 +1029,6 @@ class _Table(GbqConnector): def __init__(self, project_id, dataset_id, reauth=False, verbose=False, private_key=None): - try: - from googleapiclient.errors import HttpError - except ImportError: - from apiclient.errors import HttpError - self.http_error = HttpError self.dataset_id = dataset_id super(_Table, self).__init__(project_id, reauth, verbose, private_key) @@ -1181,18 +1045,16 @@ def exists(self, table_id): boolean true if table exists, otherwise false """ + from google.api_core.exceptions import NotFound + table_ref = self.client.dataset(self.dataset_id).table(table_id) try: - self.service.tables().get( - projectId=self.project_id, - datasetId=self.dataset_id, - tableId=table_id).execute() + self.client.get_table(table_ref) return True + except NotFound: + return False except self.http_error as ex: - if ex.resp.status == 404: - return False - else: - self.process_http_error(ex) + self.process_http_error(ex) def create(self, table_id, schema): """ Create a table in Google BigQuery given a table and schema @@ -1205,6 +1067,8 @@ def create(self, table_id, schema): Use the generate_bq_schema to generate your table schema from a dataframe. """ + from google.cloud.bigquery import SchemaField + from google.cloud.bigquery import Table if self.exists(table_id): raise TableCreationError("Table {0} already " @@ -1215,20 +1079,20 @@ def create(self, table_id, schema): _Dataset(self.project_id, private_key=self.private_key).create(self.dataset_id) - body = { - 'schema': schema, - 'tableReference': { - 'tableId': table_id, - 'projectId': self.project_id, - 'datasetId': self.dataset_id - } - } + table_ref = self.client.dataset(self.dataset_id).table(table_id) + table = Table(table_ref) + + for field in schema['fields']: + if 'mode' not in field: + field['mode'] = 'NULLABLE' + + table.schema = [ + SchemaField.from_api_repr(field) + for field in schema['fields'] + ] try: - self.service.tables().insert( - projectId=self.project_id, - datasetId=self.dataset_id, - body=body).execute() + self.client.create_table(table) except self.http_error as ex: self.process_http_error(ex) @@ -1240,30 +1104,25 @@ def delete(self, table_id): table : str Name of table to be deleted """ + from google.api_core.exceptions import NotFound if not self.exists(table_id): raise NotFoundException("Table does not exist") + table_ref = self.client.dataset(self.dataset_id).table(table_id) try: - self.service.tables().delete( - datasetId=self.dataset_id, - projectId=self.project_id, - tableId=table_id).execute() - except self.http_error as ex: + self.client.delete_table(table_ref) + except NotFound: # Ignore 404 error which may occur if table already deleted - if ex.resp.status != 404: - self.process_http_error(ex) + pass + except self.http_error as ex: + self.process_http_error(ex) class _Dataset(GbqConnector): def __init__(self, project_id, reauth=False, verbose=False, private_key=None): - try: - from googleapiclient.errors import HttpError - except ImportError: - from apiclient.errors import HttpError - self.http_error = HttpError super(_Dataset, self).__init__(project_id, reauth, verbose, private_key) @@ -1280,17 +1139,15 @@ def exists(self, dataset_id): boolean true if dataset exists, otherwise false """ + from google.api_core.exceptions import NotFound try: - self.service.datasets().get( - projectId=self.project_id, - datasetId=dataset_id).execute() + self.client.get_dataset(self.client.dataset(dataset_id)) return True + except NotFound: + return False except self.http_error as ex: - if ex.resp.status == 404: - return False - else: - self.process_http_error(ex) + self.process_http_error(ex) def datasets(self): """ Return a list of datasets in Google BigQuery @@ -1306,32 +1163,15 @@ def datasets(self): """ dataset_list = [] - next_page_token = None - first_query = True - - while first_query or next_page_token: - first_query = False - try: - list_dataset_response = self.service.datasets().list( - projectId=self.project_id, - pageToken=next_page_token).execute() - - dataset_response = list_dataset_response.get('datasets') - if dataset_response is None: - dataset_response = [] - - next_page_token = list_dataset_response.get('nextPageToken') - - if dataset_response is None: - dataset_response = [] + try: + dataset_response = self.client.list_datasets() - for row_num, raw_row in enumerate(dataset_response): - dataset_list.append( - raw_row['datasetReference']['datasetId']) + for row in dataset_response: + dataset_list.append(row.dataset_id) - except self.http_error as ex: - self.process_http_error(ex) + except self.http_error as ex: + self.process_http_error(ex) return dataset_list @@ -1343,22 +1183,16 @@ def create(self, dataset_id): dataset : str Name of dataset to be written """ + from google.cloud.bigquery import Dataset if self.exists(dataset_id): raise DatasetCreationError("Dataset {0} already " "exists".format(dataset_id)) - body = { - 'datasetReference': { - 'projectId': self.project_id, - 'datasetId': dataset_id - } - } + dataset = Dataset(self.client.dataset(dataset_id)) try: - self.service.datasets().insert( - projectId=self.project_id, - body=body).execute() + self.client.create_dataset(dataset) except self.http_error as ex: self.process_http_error(ex) @@ -1370,20 +1204,20 @@ def delete(self, dataset_id): dataset : str Name of dataset to be deleted """ + from google.api_core.exceptions import NotFound if not self.exists(dataset_id): raise NotFoundException( "Dataset {0} does not exist".format(dataset_id)) try: - self.service.datasets().delete( - datasetId=dataset_id, - projectId=self.project_id).execute() + self.client.delete_dataset(self.client.dataset(dataset_id)) - except self.http_error as ex: + except NotFound: # Ignore 404 error which may occur if dataset already deleted - if ex.resp.status != 404: - self.process_http_error(ex) + pass + except self.http_error as ex: + self.process_http_error(ex) def tables(self, dataset_id): """ List tables in the specific dataset in Google BigQuery @@ -1400,28 +1234,15 @@ def tables(self, dataset_id): """ table_list = [] - next_page_token = None - first_query = True - - while first_query or next_page_token: - first_query = False - - try: - list_table_response = self.service.tables().list( - projectId=self.project_id, - datasetId=dataset_id, - pageToken=next_page_token).execute() - table_response = list_table_response.get('tables') - next_page_token = list_table_response.get('nextPageToken') - - if not table_response: - return table_list + try: + table_response = self.client.list_dataset_tables( + self.client.dataset(dataset_id)) - for row_num, raw_row in enumerate(table_response): - table_list.append(raw_row['tableReference']['tableId']) + for row in table_response: + table_list.append(row.table_id) - except self.http_error as ex: - self.process_http_error(ex) + except self.http_error as ex: + self.process_http_error(ex) return table_list diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py index 62b72dbc884f..6a2b8480cf57 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -193,9 +193,9 @@ def test_should_be_able_to_get_valid_credentials(self): credentials = self.sut.get_credentials() assert credentials.valid - def test_should_be_able_to_get_a_bigquery_service(self): - bigquery_service = self.sut.get_service() - assert bigquery_service is not None + def test_should_be_able_to_get_a_bigquery_client(self): + bigquery_client = self.sut.get_client() + assert bigquery_client is not None def test_should_be_able_to_get_schema_from_query(self): schema, pages = self.sut.run_query('SELECT 1') @@ -256,9 +256,9 @@ def test_should_be_able_to_get_valid_credentials(self): credentials = self.sut.get_credentials() assert credentials.valid - def test_should_be_able_to_get_a_bigquery_service(self): - bigquery_service = self.sut.get_service() - assert bigquery_service is not None + def test_should_be_able_to_get_a_bigquery_client(self): + bigquery_client = self.sut.get_client() + assert bigquery_client is not None def test_should_be_able_to_get_schema_from_query(self): schema, pages = self.sut.run_query('SELECT 1') @@ -287,9 +287,9 @@ def test_should_be_able_to_get_valid_credentials(self): credentials = self.sut.get_credentials() assert credentials.valid - def test_should_be_able_to_get_a_bigquery_service(self): - bigquery_service = self.sut.get_service() - assert bigquery_service is not None + def test_should_be_able_to_get_a_bigquery_client(self): + bigquery_client = self.sut.get_client() + assert bigquery_client is not None def test_should_be_able_to_get_schema_from_query(self): schema, pages = self.sut.run_query('SELECT 1') @@ -977,8 +977,6 @@ def test_upload_data(self): gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), chunksize=10000, private_key=_get_private_key_path()) - sleep(30) # <- Curses Google!!! - result = gbq.read_gbq("SELECT COUNT(*) AS num_rows FROM {0}" .format(self.destination_table + test_id), project_id=_get_project_id(), @@ -1015,8 +1013,6 @@ def test_upload_data_if_table_exists_append(self): gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), if_exists='append', private_key=_get_private_key_path()) - sleep(30) # <- Curses Google!!! - result = gbq.read_gbq("SELECT COUNT(*) AS num_rows FROM {0}" .format(self.destination_table + test_id), project_id=_get_project_id(), @@ -1046,8 +1042,6 @@ def test_upload_subset_columns_if_table_exists_append(self): self.destination_table + test_id, _get_project_id(), if_exists='append', private_key=_get_private_key_path()) - sleep(30) # <- Curses Google!!! - result = gbq.read_gbq("SELECT COUNT(*) AS num_rows FROM {0}" .format(self.destination_table + test_id), project_id=_get_project_id(), @@ -1080,8 +1074,6 @@ def test_upload_data_if_table_exists_replace(self): _get_project_id(), if_exists='replace', private_key=_get_private_key_path()) - sleep(30) # <- Curses Google!!! - result = gbq.read_gbq("SELECT COUNT(*) AS num_rows FROM {0}" .format(self.destination_table + test_id), project_id=_get_project_id(), @@ -1255,10 +1247,14 @@ def test_verify_schema_ignores_field_mode(self): def test_retrieve_schema(self): # Issue #24 schema function returns the schema in biquery test_id = "15" - test_schema = {'fields': [{'name': 'A', 'type': 'FLOAT'}, - {'name': 'B', 'type': 'FLOAT'}, - {'name': 'C', 'type': 'STRING'}, - {'name': 'D', 'type': 'TIMESTAMP'}]} + test_schema = { + 'fields': [ + {'name': 'A', 'type': 'FLOAT', 'mode': 'NULLABLE'}, + {'name': 'B', 'type': 'FLOAT', 'mode': 'NULLABLE'}, + {'name': 'C', 'type': 'STRING', 'mode': 'NULLABLE'}, + {'name': 'D', 'type': 'TIMESTAMP', 'mode': 'NULLABLE'} + ] + } self.table.create(TABLE_ID + test_id, test_schema) actual = self.sut.schema(self.dataset_prefix + "1", TABLE_ID + test_id) @@ -1415,8 +1411,6 @@ def test_upload_data(self): gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), chunksize=10000) - sleep(30) # <- Curses Google!!! - result = gbq.read_gbq("SELECT COUNT(*) AS num_rows FROM {0}".format( self.destination_table + test_id), project_id=_get_project_id()) @@ -1473,8 +1467,6 @@ def test_upload_data(self): gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), chunksize=10000, private_key=_get_private_key_contents()) - sleep(30) # <- Curses Google!!! - result = gbq.read_gbq("SELECT COUNT(*) as num_rows FROM {0}".format( self.destination_table + test_id), project_id=_get_project_id(), diff --git a/packages/pandas-gbq/requirements.txt b/packages/pandas-gbq/requirements.txt index c72b5a5a7de2..88cf967a0b42 100644 --- a/packages/pandas-gbq/requirements.txt +++ b/packages/pandas-gbq/requirements.txt @@ -1,6 +1,4 @@ pandas -httplib2 -google-api-python-client google-auth -google-auth-httplib2 google-auth-oauthlib +google-cloud-bigquery diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index df3cd85d1ee4..86a40c5eb14d 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -19,11 +19,9 @@ def readme(): INSTALL_REQUIRES = [ 'pandas', - 'httplib2>=0.9.2', - 'google-api-python-client>=1.6.0', 'google-auth>=1.0.0', - 'google-auth-httplib2>=0.0.1', 'google-auth-oauthlib>=0.0.1', + 'google-cloud-bigquery>=0.28.0', ] From 9e78453ca7540209bf7fdcad2a1ddd067d631229 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 20 Dec 2017 17:07:26 -0800 Subject: [PATCH 089/519] BLD: Use quiet flag for conda commands. (#99) To avoid `CondaError: BlockingIOError` --- packages/pandas-gbq/.travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/pandas-gbq/.travis.yml b/packages/pandas-gbq/.travis.yml index 9beb94236515..81417d973633 100644 --- a/packages/pandas-gbq/.travis.yml +++ b/packages/pandas-gbq/.travis.yml @@ -22,21 +22,21 @@ install: - conda config --add channels conda-forge - conda update -q conda - conda info -a - - conda create -n test-environment python=$PYTHON + - conda create -q -n test-environment python=$PYTHON - source activate test-environment - if [[ "$PANDAS" == "MASTER" ]]; then - conda install numpy pytz python-dateutil; + conda install -q numpy pytz python-dateutil; PRE_WHEELS="https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com"; pip install --pre --upgrade --timeout=60 -f $PRE_WHEELS pandas; else - conda install pandas=$PANDAS; + conda install -q pandas=$PANDAS; fi - pip install coverage pytest pytest-cov flake8 codecov - REQ="ci/requirements-${PYTHON}-${PANDAS}" - if [ -f "$REQ.pip" ]; then pip install -r "$REQ.pip"; else - conda install --file "$REQ.conda"; + conda install -q --file "$REQ.conda"; fi - conda list - python setup.py install From e556f960d515409ae13659b729853016bebf73da Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 21 Dec 2017 11:01:29 -0800 Subject: [PATCH 090/519] DOC: link to issue 7 for to_gbq changes (#100) --- packages/pandas-gbq/docs/source/changelog.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index b6684582acae..f755dc8038a0 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -5,8 +5,7 @@ Changelog ------------------ - Use the `google-cloud-bigquery `__ library for API calls. The ``google-cloud-bigquery`` package is a new dependency, and dependencies on ``google-api-python-client`` and ``httplib2`` are removed. See the `installation guide `__ for more details. (:issue:`93`) -- :func:`to_gbq` now uses a load job instead of the streaming API. (:issue:`75`) -- Remove ``StreamingInsertError`` class, as it is no longer used by :func:`to_gbq`. (:issue:`75`) +- :func:`to_gbq` now uses a load job instead of the streaming API. Remove ``StreamingInsertError`` class, as it is no longer used by :func:`to_gbq`. (:issue:`7`, :issue:`75`) 0.2.1 / 2017-11-27 ------------------ From df6462a4d750b7f585cff88654e5fbb4faeb6f2d Mon Sep 17 00:00:00 2001 From: "Jason Q. Ng" Date: Tue, 2 Jan 2018 13:20:55 -0500 Subject: [PATCH 091/519] BUG: Fix bug in type conversion of arrays; add array and struct tests (#101) * Fix array bug in type conversion; add array and struct tests * Update changelog --- packages/pandas-gbq/docs/source/changelog.rst | 1 + packages/pandas-gbq/pandas_gbq/gbq.py | 5 +- .../pandas-gbq/pandas_gbq/tests/test_gbq.py | 50 +++++++++++++++++++ 3 files changed, 54 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index f755dc8038a0..78c4d6e4a5bf 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -5,6 +5,7 @@ Changelog ------------------ - Use the `google-cloud-bigquery `__ library for API calls. The ``google-cloud-bigquery`` package is a new dependency, and dependencies on ``google-api-python-client`` and ``httplib2`` are removed. See the `installation guide `__ for more details. (:issue:`93`) +- Structs and arrays are now named properly (:issue:`23`) and BigQuery functions like ``array_agg`` no longer run into errors during type conversion (:issue:`22`). - :func:`to_gbq` now uses a load job instead of the streaming API. Remove ``StreamingInsertError`` class, as it is no longer used by :func:`to_gbq`. (:issue:`7`, :issue:`75`) 0.2.1 / 2017-11-27 diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 46a246e528e6..77efe100eff4 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -867,11 +867,12 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, ) # cast BOOLEAN and INTEGER columns from object to bool/int - # if they dont have any nulls + # if they dont have any nulls AND field mode is not repeated (i.e., array) type_map = {'BOOLEAN': bool, 'INTEGER': int} for field in schema['fields']: if field['type'].upper() in type_map and \ - final_df[field['name']].notnull().all(): + final_df[field['name']].notnull().all() and \ + field['mode'] != 'repeated': final_df[field['name']] = \ final_df[field['name']].astype(type_map[field['type'].upper()]) diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py index 6a2b8480cf57..75274d972afe 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -920,6 +920,56 @@ def test_query_response_bytes(self): assert self.gbq_connector.sizeof_fmt(1.208926E24) == "1.0 YB" assert self.gbq_connector.sizeof_fmt(1.208926E28) == "10000.0 YB" + def test_struct(self): + query = """SELECT 1 int_field, + STRUCT("a" as letter, 1 as num) struct_field""" + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path(), + dialect='standard') + tm.assert_frame_equal(df, DataFrame([[1, {"letter": "a", "num": 1}]], + columns=["int_field", "struct_field"])) + + def test_array(self): + query = """select ["a","x","b","y","c","z"] as letters""" + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path(), + dialect='standard') + tm.assert_frame_equal(df, DataFrame([[["a", "x", "b", "y", "c", "z"]]], + columns=["letters"])) + + def test_array_length_zero(self): + query = """WITH t as ( + SELECT "a" letter, [""] as array_field + UNION ALL + SELECT "b" letter, [] as array_field) + + select letter, array_field, array_length(array_field) len + from t + order by letter ASC""" + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path(), + dialect='standard') + tm.assert_frame_equal(df, DataFrame([["a", [""], 1], ["b", [], 0]], + columns=["letter", "array_field", "len"])) + + def test_array_agg(self): + query = """WITH t as ( + SELECT "a" letter, 1 num + UNION ALL + SELECT "b" letter, 2 num + UNION ALL + SELECT "a" letter, 3 num) + + select letter, array_agg(num order by num ASC) numbers + from t + group by letter + order by letter ASC""" + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path(), + dialect='standard') + tm.assert_frame_equal(df, DataFrame([["a", [1, 3]], ["b", [2]]], + columns=["letter", "numbers"])) + class TestToGBQIntegrationWithServiceAccountKeyPath(object): # Changes to BigQuery table schema may take up to 2 minutes as of May 2015 From e81180bc6bb97b9ad37f1f171eaa598cb1a0db9b Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 3 Jan 2018 09:44:22 -0800 Subject: [PATCH 092/519] DOC: Add release date for 0.3.0 to changelog. (#104) Preparing for a release today. --- packages/pandas-gbq/docs/source/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 78c4d6e4a5bf..1cd36a41ed0e 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,7 +1,7 @@ Changelog ========= -0.3.0 / 2017-??-?? +0.3.0 / 2018-01-03 ------------------ - Use the `google-cloud-bigquery `__ library for API calls. The ``google-cloud-bigquery`` package is a new dependency, and dependencies on ``google-api-python-client`` and ``httplib2`` are removed. See the `installation guide `__ for more details. (:issue:`93`) From e3115fb8e17411c0cdf5b0f70baf2daa8d7d290d Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+maxim-lian@users.noreply.github.com> Date: Wed, 17 Jan 2018 15:25:20 -0500 Subject: [PATCH 093/519] Encode before uploading (#108) * encode before uploading * set py file coding for py2 * lint * move test to travis test class * try forcing utf-8 encoding * add test * correct expected sizes * test data matches * test unicode locally * Py2/Py3 compat * typo * what's new --- packages/pandas-gbq/docs/source/changelog.rst | 7 ++ packages/pandas-gbq/pandas_gbq/gbq.py | 8 +- .../pandas-gbq/pandas_gbq/tests/test_gbq.py | 111 ++++++++++++++++++ 3 files changed, 124 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 1cd36a41ed0e..5d1bb98b93d7 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,6 +1,13 @@ Changelog ========= + +0.3.1 / [TBD] +------------------ + +- Fix an issue where Unicode couldn't be uploaded in Python 2 (:issue:`93`) + + 0.3.0 / 2018-01-03 ------------------ diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 77efe100eff4..67d5ea515ee0 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -559,7 +559,7 @@ def run_query(self, query, **kwargs): def load_data(self, dataframe, dataset_id, table_id, chunksize): from google.cloud.bigquery import LoadJobConfig - from six import StringIO + from six import BytesIO destination_table = self.client.dataset(dataset_id).table(table_id) job_config = LoadJobConfig() @@ -581,7 +581,11 @@ def load_data(self, dataframe, dataset_id, table_id, chunksize): self._print("\rLoad is {0}% Complete".format( ((total_rows - remaining_rows) * 100) / total_rows)) - body = StringIO('{}\n'.format('\n'.join(rows))) + body = '{}\n'.format('\n'.join(rows)) + if isinstance(body, bytes): + body = body.decode('utf-8') + body = body.encode('utf-8') + body = BytesIO(body) try: self.client.load_table_from_file( diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py index 75274d972afe..27f991d7dba1 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + import pytest import re @@ -7,6 +9,7 @@ import os from random import randint import logging +import sys import numpy as np @@ -1154,6 +1157,61 @@ def test_google_upload_errors_should_raise_exception(self): gbq.to_gbq(bad_df, self.destination_table + test_id, _get_project_id(), private_key=_get_private_key_path()) + def test_upload_chinese_unicode_data(self): + test_id = "2" + test_size = 6 + df = DataFrame(np.random.randn(6, 4), index=range(6), + columns=list('ABCD')) + df['s'] = u'信用卡' + + gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), + chunksize=10000) + + result_df = gbq.read_gbq("SELECT * FROM {0}".format( + self.destination_table + test_id), + project_id=_get_project_id()) + + assert len(result_df) == test_size + + pytest.skipif( + sys.version_info.major < 3, + reason='Unicode comparison in Py2 not working') + + result = result_df['s'].sort_values() + expected = df['s'].sort_values() + + tm.assert_numpy_array_equal(expected.values, result.values) + + def test_upload_other_unicode_data(self): + test_id = "3" + test_size = 3 + df = DataFrame({ + 's': ['Skywalker™', 'lego', 'hülle'], + 'i': [200, 300, 400], + 'd': [ + '2017-12-13 17:40:39', '2017-12-13 17:40:39', + '2017-12-13 17:40:39' + ] + }) + + gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), + chunksize=10000) + + result_df = gbq.read_gbq("SELECT * FROM {0}".format( + self.destination_table + test_id), + project_id=_get_project_id()) + + assert len(result_df) == test_size + + pytest.skipif( + sys.version_info.major < 3, + reason='Unicode comparison in Py2 not working') + + result = result_df['s'].sort_values() + expected = df['s'].sort_values() + + tm.assert_numpy_array_equal(expected.values, result.values) + def test_generate_schema(self): df = tm.makeMixedDataFrame() schema = gbq._generate_bq_schema(df) @@ -1467,6 +1525,59 @@ def test_upload_data(self): assert result['num_rows'][0] == test_size + def test_upload_chinese_unicode_data(self): + test_id = "2" + test_size = 6 + df = DataFrame(np.random.randn(6, 4), index=range(6), + columns=list('ABCD')) + df['s'] = u'信用卡' + + gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), + chunksize=10000) + + result_df = gbq.read_gbq("SELECT * FROM {0}".format( + self.destination_table + test_id), + project_id=_get_project_id()) + + assert len(result_df) == test_size + + if sys.version_info.major < 3: + pytest.skip(msg='Unicode comparison in Py2 not working') + + result = result_df['s'].sort_values() + expected = df['s'].sort_values() + + tm.assert_numpy_array_equal(expected.values, result.values) + + def test_upload_other_unicode_data(self): + test_id = "3" + test_size = 3 + df = DataFrame({ + 's': ['Skywalker™', 'lego', 'hülle'], + 'i': [200, 300, 400], + 'd': [ + '2017-12-13 17:40:39', '2017-12-13 17:40:39', + '2017-12-13 17:40:39' + ] + }) + + gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), + chunksize=10000) + + result_df = gbq.read_gbq("SELECT * FROM {0}".format( + self.destination_table + test_id), + project_id=_get_project_id()) + + assert len(result_df) == test_size + + if sys.version_info.major < 3: + pytest.skip(msg='Unicode comparison in Py2 not working') + + result = result_df['s'].sort_values() + expected = df['s'].sort_values() + + tm.assert_numpy_array_equal(expected.values, result.values) + class TestToGBQIntegrationWithServiceAccountKeyContents(object): # Changes to BigQuery table schema may take up to 2 minutes as of May 2015 From 9f9a3540fbfc2d7454740d5be79c4c99388de63f Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 19 Jan 2018 10:23:33 -0800 Subject: [PATCH 094/519] TST: Use private key for unicode tests (#111) * remove upload tests from Local Auth test class * Add private key to unicode tests. * Add private key to to_gbq in tests, too. * Fix lint --- .../pandas-gbq/pandas_gbq/tests/test_gbq.py | 89 +++++------------- packages/pandas-gbq/prof/combined.prof | Bin 0 -> 206840 bytes .../test_upload_chinese_unicode_data.prof | Bin 0 -> 206840 bytes 3 files changed, 21 insertions(+), 68 deletions(-) create mode 100644 packages/pandas-gbq/prof/combined.prof create mode 100644 packages/pandas-gbq/prof/test_upload_chinese_unicode_data.prof diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py index 27f991d7dba1..78928a60f0c3 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -1164,18 +1164,21 @@ def test_upload_chinese_unicode_data(self): columns=list('ABCD')) df['s'] = u'信用卡' - gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), - chunksize=10000) - - result_df = gbq.read_gbq("SELECT * FROM {0}".format( - self.destination_table + test_id), - project_id=_get_project_id()) + gbq.to_gbq( + df, self.destination_table + test_id, + _get_project_id(), + private_key=_get_private_key_path(), + chunksize=10000) + + result_df = gbq.read_gbq( + "SELECT * FROM {0}".format(self.destination_table + test_id), + project_id=_get_project_id(), + private_key=_get_private_key_path()) assert len(result_df) == test_size - pytest.skipif( - sys.version_info.major < 3, - reason='Unicode comparison in Py2 not working') + if sys.version_info.major < 3: + pytest.skip(msg='Unicode comparison in Py2 not working') result = result_df['s'].sort_values() expected = df['s'].sort_values() @@ -1194,18 +1197,21 @@ def test_upload_other_unicode_data(self): ] }) - gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), - chunksize=10000) + gbq.to_gbq( + df, self.destination_table + test_id, + _get_project_id(), + private_key=_get_private_key_path(), + chunksize=10000) result_df = gbq.read_gbq("SELECT * FROM {0}".format( self.destination_table + test_id), - project_id=_get_project_id()) + project_id=_get_project_id(), + private_key=_get_private_key_path()) assert len(result_df) == test_size - pytest.skipif( - sys.version_info.major < 3, - reason='Unicode comparison in Py2 not working') + if sys.version_info.major < 3: + pytest.skip(msg='Unicode comparison in Py2 not working') result = result_df['s'].sort_values() expected = df['s'].sort_values() @@ -1525,59 +1531,6 @@ def test_upload_data(self): assert result['num_rows'][0] == test_size - def test_upload_chinese_unicode_data(self): - test_id = "2" - test_size = 6 - df = DataFrame(np.random.randn(6, 4), index=range(6), - columns=list('ABCD')) - df['s'] = u'信用卡' - - gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), - chunksize=10000) - - result_df = gbq.read_gbq("SELECT * FROM {0}".format( - self.destination_table + test_id), - project_id=_get_project_id()) - - assert len(result_df) == test_size - - if sys.version_info.major < 3: - pytest.skip(msg='Unicode comparison in Py2 not working') - - result = result_df['s'].sort_values() - expected = df['s'].sort_values() - - tm.assert_numpy_array_equal(expected.values, result.values) - - def test_upload_other_unicode_data(self): - test_id = "3" - test_size = 3 - df = DataFrame({ - 's': ['Skywalker™', 'lego', 'hülle'], - 'i': [200, 300, 400], - 'd': [ - '2017-12-13 17:40:39', '2017-12-13 17:40:39', - '2017-12-13 17:40:39' - ] - }) - - gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), - chunksize=10000) - - result_df = gbq.read_gbq("SELECT * FROM {0}".format( - self.destination_table + test_id), - project_id=_get_project_id()) - - assert len(result_df) == test_size - - if sys.version_info.major < 3: - pytest.skip(msg='Unicode comparison in Py2 not working') - - result = result_df['s'].sort_values() - expected = df['s'].sort_values() - - tm.assert_numpy_array_equal(expected.values, result.values) - class TestToGBQIntegrationWithServiceAccountKeyContents(object): # Changes to BigQuery table schema may take up to 2 minutes as of May 2015 diff --git a/packages/pandas-gbq/prof/combined.prof b/packages/pandas-gbq/prof/combined.prof new file mode 100644 index 0000000000000000000000000000000000000000..d307f0720732b8ae4cad63b1b0d3eb4e564ffc7f GIT binary patch literal 206840 zcmc${37nio(LWwYHpk{BB#>|=Tp>WRIUocGJKTpH2?6A0nBARy@+7-6L(gokMG#N~ zQ4U2wL@q(i3%mjXA~$(OL=m}>K{N=811MJj2_V1kx9jQd=b4@zHvC!t`}y$D*{$xb zuCA`GuBxv7XiERJ>z!8Vho2Q&&+W{X7UtU~PHxM#=G(H9C+6EHPHtIV$`(;+YJpmaTDK)9H_@DuF6$G$(>{hZE}Ev)Rn zcF%S0q)T<^UNfqgWp~^5M|Y!6>B^U~>2{UxOzVM88Bo~~v`~Zb2~syadhxS=SwCe9 z=s2~`9WN}E3QesYVK!Gnzm-*Jxjei;G|~&RnKlpxosMUx!`bQPGgjoT*)dHJhZziWL9r=p%VpT_o$O%%KIm;I%cI%b!p)94bX3=y&9;Ev!d~=-IW0C7GP$-) zacZ%YY3ayT&1o?#Wv3Q0t&1~3w%8Qp^8r9IU8RLh7Hk77)3O(xv?3Y%ZP6wH5MVu9Q~UQ@XOn5~xUG zs!fRLk*|z8rMUg`&9*RoN`IvX0-o--c1ydk7NLormet0n5bgx74Bi>A>abfD?U@RW zJ7|adu6gU+RJR&}cBTx_93pIT_wxVQCwKM^slXIg_E(Jn3XkM6YD^b1?OEWOX~R<5 z6u;7+;EltUUb*bCaVc9cwLau(7?v(#nCX01snCT1#(HZ%f$a0poYTiN zVj#25UGd;gc0D)MH#o)lebG|A?o%L1d7xgGS&)eWMnSsu3*K7z@;SHv`;}%}07}6Z zPo~21Offej4I*`QU_}%E&7~~+)5@?Btc-)AH$D0!iZ^c%2 z;i*Mk#ZnI|FE&hRZ?$P}b#phy+0r-TEU8Sch=J*OxQJsN0{JpC+J@>Ol^mqowpFpD z!a@(Dtn7>~)Es<*hHEEh7wj`MWeeS^0lx=j`R5*BlASur77HC=Pm~Rhq)g){xb=+E zE#dXkQ?>xgDso@tyX5$WQy6@J8=!=(Q0`#MydCMmf1TQ0i^ruq0_`b z)^2MD!rei}5M*;%l_}*F{%IA%3Z!m7wVxXY)}`zhGg4eft-Tn>I_edLj;RLJd-CuR?fp}Uk;cSDrckhB?FhBxu)_>D9l`As zpWwc;SKs-qAABig3moR0UJNriU9I`Ta!%Hh#C=4%Et@UyS84W)tzrY>e;7Vi5wZoW zp@Z><%EzoZG`$!#)0vKN33r%jtVSIslG*PV;r&jt+8@6L2(=S+1VQkh{;`EA{nuuC zF_3kfuQZf?C{InDnJ|YDhhw(#^-tIN_howg80o?FF%tvz`SJI-1(E**1L`Q0%bG z4u@dN!|4EP4D#k7>Tdi5n{K`Dhi8u4GGz;x{1X6Qd5D#UrE}S(>CSAi2!$Ku<5@y9 z#Re_yUvG}2{p;Mi*IlF(p&DeV$b`>w&V~lr5~fXsx%ND0Lq$eq@v-`WEda}b_(M3C zI@u6NA|rt+SV8^NLRJ#}`4OW(Khg~D+v{@@G$H{p{R_l&c~CJ6r9HDWK}%0|gC?b3 zbhwV}URWG}v5WaG)e4m>y#z8PuW<0?{0fVOSCX2;)0yHC*t>JVA+qGD$*UYmQ|l0v zlrTlrYEEK7%PD>_&EMibQe--N8`6=7N`=iWgPG&pDAhwxs(?R)Z*bUasaD_J0ofoVYP+ z4bLuvj?`L0<+kjy-c{TQ78C4v3-g_MFlM-dGLL*1XU3b29(6NiULTka!~i0MfbO1E zxr`h@x51uW9!ljt-J%awHLyze{4)x7Mf_65GmQOUo`vmi{J*e4gOHsgW!V;nbN{ET11`6 z_=UQhb-a@x22$(=xz=$hMTUabj(k^JQ%e|tzG`{GM7D-FZkSg2rRh>vp(8R@E-F$< zs%%btud0c!;#j~tG*HPI-rHprO`T|kd$771A8UfJ1&q26PF-S)Xeuo)WShEj;S$P^ z#|LAaa}rI!%QlNPY2W)@C4X?jR*`y^pw#6_qlbX@w$EgfSD! zMwa+G?f_}^PfW6`J>|cJujmt$E7t&t^0)|H`L$J{z^WVJ&+2FR)Y|LRjrc?59%-Sa zE9_{C6qQ&D^)`vbX>hGGafncrlW+BDu(A^|x{Vl#RxmgfeB3$j!ax3N1kAQu;P;@i zQ^q}GpQ7vph!fPp--*3Tq}2_8EDu-N06Jn;Ve88;PPA>^6~cN$s!q2P?K!@XkBmg# zU}m8UMNfg%S8Rdy8hzZLnqMAJ=qf_@i0w=s+E9)NuMQ#nurxZYc|3XK;eV~aO~d=m zfhnNg`KVHPPxk}0GikxScH6b5UOf8M=D-vZv$n$7+FSEoxyUe8Jx-4aV5t!NAJ-O_ zq=F5$gjBEZ!s_Y@+sM9)i1gsJs6lBEs9Gn9^^`OtO4Z#EHB%*T*<|tC9PT3;#Mi6_^5jfa~tC5-%B8I?OK0 zXd(L%T?@mp!}E9POsPaiS-p;v!`O5#B|vFvn41{6HJzIYUd+l=5K29aPjK-cZ@>4& z?`)c~g>H2}exH~;&;E@;_Ju;Jjr33`JE4>`QKbyB+Awf3v8AF98MqzGzr`omai2|J z|Lg1tDO*U8VC{BxCevI-!U{5>)X-6KMy*eVf^YZ<_PTTZ#`o6DNCi)R^y}xYJa%RZ z6Dmje-EhoV+c=jjNer8DII2@Efy)XfeSk?1I2<|%IPeopI43*n8`tfhvIVmC!5DJo zYxtB08xt8YXaiFE6ZHSy+i%XAwqwc`u;ccDohY(7nl;W<|BOazZbzoGr7Z)p#Ka>0 zqJIKY0OPE?tthfdHN`t!h8D}iK+QuSFFFpxF$^8kJ|#w~ul@4H-+e_5NCl<<3O)VyMMI)f7Oy#b5mPUu6-AdK|H zdR_y+ZuR0WuvyUWKKFo!b5ScS`QD6glZqW^p!+18I9#&Nu9uEjLE+wF~=z=VFLz zBUac-hC7aQ9Hp_$EDMWiSeBsXf9B-A#yOY@A%%lFN2nR8H3Ng#XI1?iyP}+P;hoYk z2gLeMhy*Banl7mPvgHU_DP*9|cV=4i#bxQ@@*-T`?Lfono8G>KW!(n{I_ra%-CCc; zX_hFDCW)dg+=K|k(5rDW?+aEn<^qr>C)R>OK2MWcct4q75dNNr!zV#GVa|l$=?msG zs1c#gOb*6-z2EOj;%x*4t~~=!792av(>Yj^;M-ZDdDP9YccQ=y_d)o8!PAkD zRMqhHs1Jc^f{khfSgG%#s?yulYLpUWt7SFl1N&J#KE*H@sj)i; zwMkk#2wGLP4TgdcCQTVzy99;MSs)$*Zx_Qx(+B2eJ84DOE*L}zOe=>I)1jSN6(Ed` zlbN7;jYXk`fLFsqwU$9gb^^YJ91{3T2tPr(!67hI`CKQ60zYM`1b5A6_X_)VKk-cr zkBl1GWra}Ds_Seoo~)VDUtNKU;kkN!Y3-edxQbpdWAMw!K=-tX8hRQoGpVv3F&;&G zVG$jIItb5Gzjd*whC4c&F={U=GyYf#l~W(lL_qyzlIB88c`-1<(<6v;;+_ z9(NqSpvDImw_T3v|2?#Kq2*5&EqDalZw9!K~cR^OWD%r3@sQK;iE38{wdAWe@( z+}P~U@nkNzj^=6|p`*6LNWeGMr@xrAQa1n?APY<#8+>95km9-j7Hciaw~(>872zC& ze`$dY!%)~tq3v`8Z`@$6nWz;q+_^j~s}v+Y1U9r>)N>x=8oGtlo5p$#+#Yb?s_ad2 z%*L-GyT#ik-0v$=&&xo=x4LrgIHi$RS62foW%i4cu?IHSz zonA;&PfWgI{Q4&*h}Qns!Xg;i=1|~4* zBW$Om1KwY$ z3p&TA0#hJS`p(C%FTdzmU`9F`n~pxr{Fsy(6rFQGlhZk1ej=@PwV}!CW<@H3CYKQ8 zDAL56TlJ|1Q5u9FzZEP@eTe-Z{1x$r3>~Ba0_{#>yT`I!?+g`9N6dUvyBtWRgi83d0?nC35|FBv86x9LZ4rKVZ{;^^O z1k?H+)f6>HS{w%uEs89ZEdX3=P|L8$4Y)hVM2)+sTO%jGOYle+aNyofkYsx2I0Vzv z8rg$1s583mUQi&}sdEy-G{x(IxG7k@)GGf+0-X$E1;#gM3tVc)eDTUkFg@6{ASv?H&{aB2;HPv#9S?>nGQ?n zTO4-4Vav?H2V)9TqS|X%d&9^uDyLqz`kighLpECdu_Y)VF#(aBC+7|)XY?>p;Uhp+ ze}d=6&RX@tSrb#Xz=>LSe-%b%w5!)jK{-E zSYf4^^v#pLV%?4hwzuPNm;xBd`>6s!YAH3^NB&jKBXFoO`kW4Ow?4c{SL9@XxHXA? z>z~+)?+e4V-A}bYIPU%X4Z6pLA+WffSd4iBexsIh%$_w1Y)7|6#@}!DC z>7Uq$bl8bH5x+JDTgRF;91Gg6Sy>Hm+;h`$h#;A(QERj4OA~KbDb&H3>W9Q=B(!*1 z(XnEqdll*!4ve2@d9E?Wv^C9Y1n8K+URH1%WUheJW1D6HAtUy{jmg#OLXLnYZkhHV zS4R=?Gtj6X1l6s+ia-AU!K%hVK(aR#vy95j>r1}jhrkq?uKpD;TS)Br@v-lT8X`1m zQzQH)n%AFT$>Q%1d-&ZcDO;fM+M^e{Oftq*`zoOgY5%}rAX**~-342!crl3Uh9PJ@ z#G$EEfDk|qBM3kKII)Rn8T2R0paGF0T+K)^9R|bbiwMVWK)h!rx>Zsn69#LAsZR3{l@z6Nb6CilNySv z435!1aY4KpEAvy{3f^A0)OnbI$9;vimZbxnu}Zi@0!waWJ*#G>*a6fJ4EFXrOzZLJ ztg;Odi9~sTf$PWwbpk*z9!5k;SmZ{He;l#@8}Bv;reMHt2c+3p zoC5D{*ouyNwCF%Qh1l2eWqV4dZs-HAXd?PnJ1)Q~mGd!n9CYi(lN@5~Fom3knKC!z9h%;ApHoW>- zOX=T=!@SxJ^CNTd?4xFgW08AnVZtgmVzD9G`?Dh_rik_uo;OG@9^HAb!!A(yEFhx; zgYIJ;&Sv=Wl}St@Z3pT=epcZFdS1mcKha(n3w!sE zEFL8=M{F@1kG{(rK=jh7OM66m@zRW{aw72Fa_X4$yj+6O_Yb~T?cV6 zg{ts&PsXXsWTZ||iQ4>$xhC>SvoCo(t+y8kXP=>bTt3g@Hu2Ina5~8?yV>B+YCB=< zY&LD4LIRp>+rHJ|pzhkKM1k&Y!a?^&RQq>8>BMzP4m=Yi}gBW8| zU`XnBfkkDep7dCFE11zK4+h-UOX4-#;uTl7dxIJcZ(@D>{yUR?I^o{NRA34Wf9``I zwrC$gTBVZeYAtm^cyg=Ro#SV`Id25;6zdK(!XL41+R#jM{Lh5@9j!dpPSR>;ORJb6 zp*N(Wsa1@(asd3mMDBmk+^tTl#zJ=gSSB5;Zfb{09IYZ6z|ai7z-akf{qf~aR5AQH zb!=W&PG{OO1&}GhA+i)>0x&sh$06|}NNdQr_g_!rt4N?aZs?^kiZ)u%X;BD}|>;a0hM#%Y=6TgYp2$4^*#00*|#@SUg}f zkqroQk?4FjT37top#&!94p+vyr_V|+&1M#>Y&$w4!*582ACl%3rikNtLX&)5q8G+| zVBq*W$svvWd=GFS`oje0$3K>->jmUGg79N6zeG`NgHoU$jrDTi8K%X?zWNpz*@~wi z{fP_;F^^(`jw@VBjkd5I2P9mHkFI-Ele5|*95eKBE{`NGd2%=d!5Igo14Ds}QzZeS zf2{g~)8vv~9Ev~RO-4!aRTWaAgD^0AS~Qo3=x_-!=6P1P*Z`9>bG&Cak!oVqG0%}$ zyzN=}Ek3x7q^~KPq%N@qAf&>~n?e!^rvfXiy(K+Nn&U{>fMPdKiHBxM8tN#xv&(Q_ znu5Ddr6dm1>>|`xj-!kjvKm3z6v2-(wYTZD{da)%CS3^)~TAr z%8973SnO&M6duq8L0aqEbS+N7ddbRFKr6^(W}R_2qkxKJOS@zKIIp ztn%Qw%t)0Bn)TtmJSb`xy5k|N$WQRXyB~)?+HGpe7O+CJag3%Q^(a2z1+H4T@>DAl zOJ9rlY7E?+fWLjx@c9!K9hnMD0j+C`)wckSD{{q`NcbXRKlCR^U$SJ;v|SdaY=Ke(-jjaeD?GPv zj}l1uEr@vi3Fe;t@M+@?&ZKMs4Alo0pG<Gy7 zpKI%v3^~@0e<#L`K%RvexZJuBHx$792Gdzc(&$gnG=IU#>m9Od$`(L31dsF!ndzw9 zpp$*VQ!)d;Qh(ya?Ywl&{U@9U9oZE8L};R#ecT-BGaolA=G4My-3}|<5;N9J5iP4i zl>=OFGTvw(fH(p?Z5xA5M@g6w3Wo1 zuRZyT*Qyi}BEp$U_aF7AIl!(wj7I=Cu|W3>gBG3IX>XvLLQ)Pn_Y#cQ9T}^0CIykl z*f9cgtLg)voslfKVuw&;s`(V@a41B6msTuyM%^=a7(94M$K_}2)6QxZ%T!n`dq1TJ z;U^feX8sFjA3h)ErKrFm@7IfoP5O{Tbis%PC1kmAsxZ2sJVvfTA!>kGp{QxmA{oo` zB?*A{G6{h9N~u4A7?@{nHL~A$Y8-GpwHG+9qe@?OS<|Gi(n)=>tD76~oUdO4MPm%C zRT3-L%GHVQ3oz$@;a(T z*hvoNu;0hL1m`?;&>eSOfy6haUbx5JKiZjlO1L3Imi zbv9E_aoxR|A@ZoU!D^&0UmVO`W=uLPP<%&W{#sK5o4JnVdO2)crhqZ6#c;wjHdXZz zN(t~7?Wue99Yi_9_31l2 zf6JtlEvWwZb%JWO)IYuMUIb)$q`6*e^YE)_qys-uY*{ShwYpT3DmEt`i%jXS#sH{t zGd|@};)UY6lv6M;t6r0FiZuxG4y1-W{qJRDvT5+c+qqW^VbTwMsSu-ZMj_F?-g2R| ziFLig>0Ys9JXH%@-2`IkPjL3H$Lw{*H{h`}1qv3u3Gc+|8WrD!WdqiZ1@SJ(0>K7s z3WmSjerC1}yMJIwuX{4a)*ytq52sT04i)`PM{$5KZfE-pgT6BW<8~FoaBx#A3ihg| zPW9Fn;}}v$ugt9=70W~Q&U1v>#p~2d&XJS7%)L?DH1TDyK*&#U`m_l%H@_apaIgBV^9p;~Oy%dP)C1p1vcqQz|e8)C6(+g|sd~i_`mKE?dGq8emB;g18cd=i-5= zkaCvpZsXp2{95;t$X z#w(lYK?;e@f|`#ivb-e@3=cP$LNqh#7FHXgGoi@!)Nn_3#*Zb4EhN!*;tBUrvQa=w z)dCFP+8wR&6AXBH^^-GR4O6yYP*+cC)G3|{ndJ-^OH8YQT zYfM`+Bi$TBDi4mOd~qz`8>u}0tA&M@t&g$U{y-k5(4UV5TKoi$75X=)zJ+tJDJZ@o zf^oSik1>pBS+N|uu)>rA)V;6ipJ;y}d zI1x~5{u$pFGSUybM-8T5kd-!`9r6<$?_xoj z$uYZ`h(9J%Jrf^bR3Sf6bD(mA)-Z4uS6kut;FW606I0>*$&%5}F`3opRHZI;>awb@ zXejO@`r~J!X86fwbHyJ$5pF-++Jen?TDm2lFJXY0LUZ^XPSikj0W%+l;I<2oQ>KVs zsPtPgL()`RqvRM)s+f3Z8~Chd32EHEZQM@Q|CF2!H*P(b9bwKox2tUS)#^h`mQhR5 z>c(VlZ9|ubS>(mlO<{<2>LpQei=oa6eI4`*`3VO7?8I{6nC(-x(5?9P0DA1=4Dl08 zYQFj9`}c(2Y6|Kk0jWkRcIE-y-(@EY09oa}sZ&esAq~5^G0V*^Ks2nk3eHVJuZT%( z;ey1wA7aJ39F@xBsqko14>LjN7#C+(a6S7+1y5V|87JC2+?w*Y>20 z6S!CgDWP=5NHdSs!7A;8K>gUW5h!A@>MM!QlZ)a?q_-ZDtfw{4t+C1L=P;PqPCNoO zkZ2*t#g88WhON5~62>MB(J&sHN31IQ0g3V$IA6^#X@?>|VhK9+x`u+GFa3oFlcPr% zrBD*+qNZbGZ%6N&d8-;gg>DY$B ztY1Y&!`EEd;YK`}@Dn_6OYn#6BA9ec!EWJ~5i@=wgBenriwSKo;vhfA13lUp9=pUE zP9tF(cwpugs6Fs6$$cUlLQHh^VrNj>XZEJlyjb-=Ma z8hQSBkInrlfu+ycks9sxBG0U<4imO@Al}ZYyt9sg#H@P}XP%+J>PpwbWKrG#njoBcT<{^{J)snmO`VmiA< zi^vFznGWnsH5Y5lZQ$80x{cGL+HVl^CMElHDn8$*$wbMxt5kf&IBWfh-XX>^lE4JK z<){83>!^-@4=X_1O<%?wG826#qC%+_G19e8HAq~oF#In%D32{br-n+^ktrb45TAz2 zV_p*tc)O5#NoIiCpFBnAEi>bqa`Wg}Ie`ZEiOSVzr96Uq9c;d;Ro}Wq|BkPXuOTo6 ztl@nUYq(dhb1|-^*rcOdqKCT9fn+XE)W4A{sCb8x+CN;&QCyY0-xVXKtkjq$)i?6Q zWObBWd^SZW4f(XOmm6?Jx`T;*czaB)kO^^;mofW57c!Wc)Qegvpqyi>iSjW0Pb8$q zSP=ZBYY*rCFhSd;qPxWA z`ms!U0!?q}V~x?I9;VnzlxSJ>QJr!rQP~sYK;Z|dtmECv!{xCQW;ql4-b~$gLVXEyi|>np4nO9AerN{!B4bx=$Wxz37hT?kiBTxNj4`p2qRHS`@1CfVJjo+;cGF2Q@~>@kH#B!QSG zW+5Z`DzX=Hx(!Mn_fE>Y@+fU0;cxSJI55=Ph%-W34m1TImQE-eXbA2q6PHp@yJ?sYC(r}GG8A;j_ri&t8?WcI^vk}xyG7~nMe|Z zU%O?`gvS=l2Yo>K@Sj*d`V%}qVB!Z0BWJ=P6TLEn6v=BpspgwB6Zch6%hhcjvMo)bh9W5!3iO$7j zmI3Y6iVS|Mf9zs`jJ<{AJ)axM2C$cC<8QXD&TAa8*@S?57UQmjXcRB=bs>xvJ5bP( zZ$Y$wbO{9fP%X4$EueqhLWwPm!V90_5*{_%!)3fc(yO}aHK0)5pe@^r+A-ZX>Nz}t zwQ>H47*3x*oN`l3F(bl~&~pIdDhoi;?Eykw@R%upM6D=Se!v0ZHXrjp>pqi=N z+CE6O9v9b<`VgJD$S`zJ;g(p45#cqKj8X_w2p4k(+~CZ95}BcH<|EAi9Yo!uEIPcs zJP?N+++qWE40+I>;Q7D*dd=6i**s+nnBns{Rbx1`{xAh5*vU+gohbv($iXVdfiuYn z37~bD0_OJ&G!@Pz9;;AFbmCFWfd2vD#@`%~J5kZYPMsxfJ6oX9s_fk|4B$1Qavj^| zJG<>jON@7G5)BJRv2_59bgfUJ@*#c+OAl9dAjdV6R?mI1-Z|{;=fg1YZ-Qp%rW!o9 z(81Gvtb)+Nrc)k-djciqbI#zUVU9#|GthIkcke(4PVR%04AGL)3D0;^hkXNUEqqiT z$0Qk(ezqh(1r!U;J5A7c=ADmuuMi)br7&}D3C0V#tmX{EAD18RxrwH z_pNXc5dLg_5JAWQT$I#}qIw+6C2DFu3x#$o?EP zFFrLL9^_i^6fLwcpR@f5TNokM3*+cYOtU$x+4S&JBGXHu0X2upMw7-Npq4h0l!rXr zhMY7AAKRn8#DyNK_j_b8F{`N8N--|B8l|`m*oFK=f%-?Jg3QdLrflu(bJd0~ zav6`pby&Q#Ag@9*MjI~RUy-}s1W53}cgIb;;bz>2O?|uj-A%ULEv3S0DWMLZy5%Z*6fkBTiu?#?qM^M* zp2T$(WDk!enl=?qBJ(xUptBFo_&Va*)$(FFF(XvL!zPk;DCJ_UC{vHqqpKbIR)<{e z@G^x1AMMh<2Ln*Ug*C3@9P?vCBW%GuXY3Rv-)S>RQ!QM9#`GuhtRbjh#M@SM9rkhN zd1F4)wlZ$Xcfl6{oe774wEi2Wwa#ArUqevv4h0tT1!nI99`|50&2R^-R()V2UR_rX(ykyaMwV@8ZiNh+v}`f`NiH_=u) zMdbLgIN1W|^bY^}B&&{tB|zZ-ip`0S5f>Rtfmez#Jg06M!)r8Qw7~ZHF1$1o>TBTz zA~`0_f9e;s!h9NU!iu}nsUsqeV!`~1On^Z+wMy1=EW$}$0ai5VcG=pYor>1ykq@;A1&H>U>QzP11c}2rDedyI;68)TE!W zq563y$pX6zajNnydxvwi3xI174aP*npWq+IT-eUjtyHuw zxP#mLV<+*%u-0xyt=i}sI+YmFWJLXh3C|Qv9lgg6j)ki` zW!T%Zutaq~mxEMT`BdqOSz2jbKSn*bdb$Zo-y3xlKlb{wEtrFmdzHfE&k3$u^~b$0 zJZtZiY6Op>xdFiRnZ|o)$$W64S;WF8)cu-I>Zy6^ z89igMa5GPOsk>!7ZWE~`JJtZGLd7rApF1;3zqRUi68ug7I8N`9hFv_LuU2x81}zN%XF&fPjQ^_tLwTp7mWzEJ#HV5iITavAkK zQTjZRfVp_mQiC`Z!=R3LB9;S*GxYnZPV?}cP&>7qe;sHiJdq%)q{T<@KgEv?Rj>t+O@n`p z;Buodc5ahl#4_uuzetN$u|jSEkMuH?|}0KJ-oF7+&u}7>C>rzUPeX3n{MJf8{q>240vo`6>EN`I=1i5`_E1;XDge(Qo0JxosF~!=? zL?s%*0ccEq!e+)vGYkWZw}P=a3!Eo<%vg5D#Ps@Z3%Y#}1W4QOB1E;7feRvH?^ zw5dt4RUVF-hN`6%=#mFfIgJXdBF*@hZk}_+htCW%U530BRUR3u7){51Xt1V5P+6aG!o#SAVQ& z-WDRthk2|fgr4cp9*b3q!{Nu;vTQ+ps&^n?1*VU=Gem{>Lla^8Z*#3ov4RFK#-2S9FNV5>oC6&kh=PS zgmCOL*mYXXY%y*Hj%T31@R1<^Uyagd&YcE77rQ8&aW*IcPXu&z^0ZNBVVIx~m%r1> z2vacF9Jw)?hjxYdA``!Zw(E>qeO*wmUMDdHO#M!1S1tY$#OMpZf84;Omt_<<@e4Ba z1PBNuA=ew8{$pC_@s8|VZsIUJ{S#kB=ioP$zv5FKjMokM*fkf1+l&me+n5@VmvzDr zC$4g%ey^zowGeq!(HqgXv2LQ3pP-7=dVC~m4TPBm=q6@EmP?HqL$PlgW$9Qh0)CEtN#Y*U}EGS*N7qaWG)jxsnAPK|LQZ=zG>@MY-^hQJz}YEVM>5mot1+Q{s87x z_wlnzz+YgQpPKX}csLr=j--_d8gio$_AOHJ%(n1s;D_kz4qd17r8Vym|kp>|8 zC%)ELIXsZ3hK-|O8+}BQXsj9$PhK19CCmo8aT=dcHMtJnK|MMDE*9_`@IUQ!vy0dk z)FPP;7g1BI#rM&oTE)Nqu!II z-Pp|1u*jOGM}o;B$(OS;%qlV;R;K(6)Z_jBE)X6{D!|JPeNz z?Y#c&YI<0yvXIarF4|4ZNR7tzqznaQdAK~t6S+mvHq$4_!$>B>q%*O4-bVYjqnOdR zN|_Wbj)q{t(hm?5=G)9Nplo$TNj0QPrnebr>^jYk!HP z19A0?h_Dn_BhX=D-h9L(m!}(>oN)TP92v&_=s@JWzFsN{Us^AvR##7uxC}Q-1^?RP z1ofvWd!*D~1>_P^h#$+Awg3u!mw&y8pH=t(bvHVT*PO3yzjOE1+oYmrn!=MXx%v~4 zdWTfdHfgs3=MUJ8XDm85!!MJa$I2rVQS_aTGYm_78T5#d)P{CBum2ageQj0lS__2O=7KJ?$lJ_dz1G}sbV#0{dGMI zC?;s-Ef4i-bt~p@VZDr?m3B92JKb<;(M`FrRYtt^4$m`VICiG*DGvr^w6zvkRG$%I z3WvhZOGI6>9zOz$bAW2t00hb##h%UMwVvq8Go^*oAVpdre!#@>0W&(Mb^1G_&tJt~ z@M9zHY#{;<_2u3nHvSZS)KNiArK!LbW@cUphJz%R&m~Pdo|r_vZ=s655>nm-L&`2| z%@*ha(C2Ha(s(OI0u{y)XMXGkXA9WAOJBS-vHGT}1Wj}xKSu|m0`$AX^se(|4cvVO zfvUH(x6DTV7JW{ApkO2(Hnq86bu^3-7*BDz@?)i-l~ODFs|TiHc^o_k7=a%!5jC71$8^LLQAawH(jjW9q_nZ^MCLvgi) z`D4o;#L<^~SFqMlUh8!>U8nGS)zH~WqO~DEtMEZD&dy<%zbP2-!+>DY(lRv!$c zRab()la;CkO_D)oqEdLeSEtiw7?q0SS$)}oH$#|dPksH$(^ z*FfC~RDX2QS+=^Pn6)jrqUwZ*QE}#4(w-W93EQT~G3si!;+sW4sajUVsd*~aJ3!n? z)FaiDC}QgxK&qcfiw@i$WKlpEJ1YrVd`@aNW>}6;-mpLg zZQuxS9#>1MsbGor!(r!t=jc~Arm{YP1h|U~vr0I_4cCI3LUP*c+}=hsBZG}Dz+osX z;}huvho^!E3;&xr``2=LVG+6x@5OxaJvda+(Jt=vA%h!Y;>8hL72R2Ip*i;4XbL?A z#3Z)F&-;3VL`3lQ2L0Tb0*%R(E_6jOhF)w6=IVfRRLV&r_VlgVURZk?r+^>3JJ^Ds zr9dX+aijY^-fZF-sBeqX?Vm?Z5q5Cg16ciIC8F{2yLID%fWdTV!E_2uYH0JLCm=2X zs2nn7@G)tj4wQxD0vsBu?JMe`=Wonid4m>;HOFRY&oDI93K6`XEA@0*b&=! zzP_kdn@P``g6<(dkriPRbN6Ty3Z_%ptUd?~VlaYEARf!$B9j?7JhY_V#;@&j54NB_6uxt&KeJKVty6tm4dSmpZ>R9s5!k)T0S)518C2pkJvkn# z-2{16)N#V(PJtAV_`Vn=C#loQl*k9kW3&3Jv{=VNZoqe=FPaY)A`d$c*lwlSb*rnhX(%`EQ;F<(M1w%KEj;Jb&?uGb_-Oh52q^* z*^dz!3O$4mPE}U|Da~-y8A6+C5W90|f2&3-0|`_fikoe0xMy_Cu#9)_4DgZSr>0-x zHPYu$?3H-N3Gt1ktiEKD!*xo1H@C)$it)ip(G%5?oa9w#3E-ny<|i`t#r7P~SF4q4 z;O$Bty)Hprk>WP$!X(HwmEK3C@_<}hNP;>w(&Nntr*JLd2a{IOCG{TpLu!Q`u3XJ*=~$f0r4^kTAl?4ApSZF^WCWpcZM`unl;6EXfqs=4B`s9PBQ<;&;QOB@z z#4gz6e~;M&h-1+n+kX^l6dZ1hv4|MRW>To0GRxpM+rd1++6;2(442_x7Uh`gMYjvc zzeDgWE3yfykhi{iQE5EArvpOCVd940OS5M7r+< z85;mo9-D4wjF^68DP8QurU~9kI)*(eOo`46FH=5w9B;W$#1$bz7KeitM(UD`!Wv zr(fAVKWaAC_&PI;7JtkZ2V%799^qf7J9-GH?F7eqAlIW-LKF#&VnW1pN++x7w9cs2 z)`Hj7XjmSO_HNiu!LulAj4@t#0tE9F38s3@?o=C5=pTYHbb{&CAo#cbakKVQ{JX|* zV`4<@D)`sonuP6VMIGBuXJ#2}Kb86DRBa0rN%X$gX!Fq5Xq~u%L4zT$CYc?_34OEC zL_!}sjwkf35z!QUNg=}IL!;n-NaaSm4j6b|9?&L!Lni=)Xh7E#dOzS$8@mq1?1uaV zjrV+a-UXdVwqXk0#>jICiNa4be{Y$Ie5=_%6n}qI@t|jx%t{5OfDzD>Sy5=9ya`^r z!WeR;5INY29a+Q!MmMhW@Cj8ACWs5hB)_?tXbWmGew`2zfYm>>$G*0XeU~za_T9a( z==3MJb-yQbYi8gXV|mhdfu8M~$@%nhs9y`Cxm{15McObGEZAq0uMR(FK5ryxa30vz zUDe>m&}V%zYJZ?!do<<<1G>@5S>EsIs3t0m zVJr{DB@4B@z>_h_><~(AG<^I83qTuFNRoMC&o@|AzALA5bGp^)=}OGstBrZ&tnwJ% z2KCd0*QuXGrcT02oI9f2JZg7vp8iCd5lq!0b%ww}DA_1gmN5L-+$pvIMt>H_h8tmV zlm|sw6+tqGrMGpPDic^`pybv`)Idl2r}j$5GpstN?9?zfQQpg#sQ=9*&D7Gfn##_+ ze&CPRpZvlDQ&WK{bgL{dMvrx15|pf$lo^SExO#dV?&{`lHAqBN9hgLSD|}H^Twa{U zc)fPyWzY$9j{||^$t}@F2Bji>bC4EQPvg&(9}?y%d@$5H@X5Y+oEy<}a;8I5Cak#Q06iB3O2GK5Ta)k&qnZ?J@<5jdg!ZRXUOw>dJqBy}rPc&M$>`6$7c$ zUvePR>_F16y)pDH{r_nwYA$9`F?Y3*=eNC%A6b^mlIE7B^o_ftZlb!pYd<%7dZ?F-45*lGoV;8@v;5 z3M>1ohgfBH1FDS9v@ruXQ_w`O*P^mJj>*Y7_JW3~yqs3uk=Ov^Ic!kXZO8T^ohg{A zx3KEiRzz$@23viWkLi>Er^;K=Ucm^Q%A;I*_XeIs+0RFAQ0C@}>-RaW z`8k%g(W0hc7F_rks8t@(+NJoMUepROcaqg(8sj*1AYLwqB(|G+1lBK0PlKsVqj@!l z%};Rc2@~#r>q{+=`cZ*s9>c{-+dS@3A(g5&na!pG%1`j_Z?+gV_k)vCwgAHF8ag&_ zU|3{eH3sC4PqlLX=o*&;g>pk!YzrB=<+xBDI)}1v3d!O5=Pel?rlO$pmc)qZvyD$p z4<^}NPtM!FI&O-LhsHmVb3|Hpnd{KfVjiPaYKLp_+ufq6;ntcSVQv?1lw^rx1{qnb&}-# zM29J)=6mnVdBu0O+zR^ZRFDSU_2(ks$3FXE3#v&#+#2=6q`xdkI}mHpH3gHBRoD((|Ft zXH_n)nuSMz5A`QFV9wNKPaJ)6$`&|tyzx+BfMmHrw+K49%|c%C9B1-ej7ve6+r1YD zuVgNm_MC2+$127cmWM{ah;r#T!QrVb>SdW07vL%xv!|Ji+0#<$c;F7#{|6X{yxuzQ z;EPV(=E4Q3z!X4pZ1}kSh{lkk45?;IOrnO7x~}E*;|Zz26hN8|R$2tds|nGd4MGeN zo&-1GPshNWWU1vgFn1MkU7vHV%&;`!csJbgMVPUzk_7oV*3-qiAy?nL-jpcN9WEZKUDVS#Pnv&5Z(kwZN=5 zxy9Wd?Im#a;Cn(OfEA%oh039kIB;jOhe#-X>?z$AETL$@)n;7l0kj2ml5Ai(OsTx& zXnh){piUML$G7G$&cE_6;Rq*Fyd2fZQEqQJ0)aB0ImX$+d50ps2})BnmK4to`Z9w zH?74T2g_n-xwShr{RhC6kbEQmTVz3*?qeh1Xj z`eLq*Zgq(CSFIVwV3Pf9;_)wo<)`t(zJ_w@_=$pCL0l1_lb}U;siS49Y&3gjBeVmt zTmBRbifc`v9Iw2iZJrP6;HYNHFr3)-v>^YiYm~0Q6#TQUw@dqHT@hGuS8jA0&Iv3} zY9vYOpD#2?wHqTjf6d=}>dNmF-%ACi(5=n`BPVEM{$yd2o%xn!-5Fj_%a`(aLovyq zp4vgl(+Agq5=r!l5bhFF+q(mtR30vDz&pT@G#IQ~93Fnj!TO0HX;{c~WT)ji(j6Iw z^R?$WVkZQRnsyfnp$UB%Zxass4tA;XB@|dJfEs8k{`ibq%-8gI+>8bUgwG7 ztR3&Iy)Zv5kLQPyLpX5$#~T-)!HWCS`3Z6x9=mwE+mB8K=^lW^g;9X7L%9LIQ|H3R{TUt#MUOi_tp>}d@db5yW`WxE8l+d z(3Xj*z!c0lf5dSPMC72jU7S%!7Hceg1vQ{=_Y1E<4aE~cg1x-kp~?B!ajtnWYT|)M z9yx!3d1qW_WuBNRO`N`L#`IacOmCe%b9!=Ji8~P1yX~gj-0#Q3cGOd(5&0ew;bS%l zL_GKeA}ie1EP-wN=(N!FNv%DK{P^+4kb8i%mUc^8M@J1 zbc~HRazsqV%y)@&cPMx;2^3bu?W&nJtya|U-Dw>9Zui&&Q-LXT`>z+j`hPzkFn+!L zQh_OeUQ}f&`{UzJp7#(*64}XifzUJJ@Z29}w1ej1-)?!h?M-&hwaw9Kb8R!{Ok0|s zAOqbaoLmP!iNkglhoy+Tj7?PyB;0Be3&$?TWwZ-sia`wE+&VDHj@{sk*(^Om?Gq=b zAslM->twXIio4Zv;gC8o$$odR14h=Z98J!3`*wt!_uK2vp4|_@`KDm|t^YpBbq7JttGuFbbPQTqM>bMR} zk|<7uRc#Y{ceP{3%(hXCWL080a97n^Ym`oPpfNKI1Sj%*15T_1lR#jtLqLn~*tGPF zfKx>5J8b{C?AagRa_?;IJ=FsJLQE20)S*ab2r@5l*Qu98%2_w6rlUuk^c51hNAi_P zuiW$L1^c9eOG+1?_56SXK@{(>s;nzE^Z{OTijJAA9Mze`cCC1P-su zeo_af9u5@dy}==zh56T?;J0V*{Js4$K`NNsG;_WC|Gx2_-R8EBJJp@BXYk~YUxQO) z+^khEoHgp?lz-c2^$vHvclqYH5VZfT2mEyWUpL-U%?1DAY`6_(Sbu_3@~8dZzGv;5 z3NHV{FJE}?$+i@PvW`gI9rBn@ZD-&nly?^rtm)a9k`!M$)bggTuihu|v^j^}C)gGn?MKQ_C%Ep)3LHJ|7|_W;vV@BZdgvv)TxnGu8+ znnDfr#Uuc=nLyQ2A&(PFcG;Rj59f&8bWH?%XM+yM+62PjmqMD` z2*XcgEx-U|O^i+%diW5kpY@2-fO#MsRJc8}>6MXsY{r`SZ3d>5oS^L=2puUD)-Z;%S8q<>41P@q; z>9A8j5Cfs$q=?`HhrZzlkcKBpZ8^o=q>@^mP97nhKoP|Su0~@iF?H`?1FM9rZhWQ( zYx6eXhht>rt$29lGuup3!>l!J+4c;w(SJto@P?V*jW~1rkJYRML}#MV$L0GdAsgAF zqLnS98@XxE#!{-hfK>Nwopt)sm+pHM7K}Pr)};eSb^T*U6A>re4v=^b z2m)8XG307k9L)Gl4N7&>ffMbv17klki90UAUnE#pmLDG9FgI1uA~VZY{s0iRK7RTlhvK+2)DPog6&F znGAF2=J|?P#vJ(An5D%WwH@_XrsW?^@6i&(}S^X z>U1nRG&A0&X)2-(ZjktGdN8)l>o~4xk&Gct_=&dl*rn)h^&9*iQ&9)Twn@beN&u31 z@#dAvcII)xv50whgOpu8(ZNH~K^+*|!H+>RnT!S<0HQz9`@^^_A?K0pL5K)x28r|l zW7}NDHuZBd4dH2EJN*gH{^_c#Ui$r)5P23Ay47`h!opdE;V1aPr$=wL>yrniY{7K< zKD*_|>i)K1yKN-Q_dqa^=V8tTKhfa~Dy@-$b|`gV*e3Al7hZQ8X! z>F;l~U)wT7M4kDquQncVA;_~805PXdm9|HLHx65R<+8`NrR>#-Xf9%tJhcM+7cBD= z>~_(~|MUIZaRlD#o;?Py_u<|aTw@Tu&2tw=#gDzJv7n-iRaT#Xde}tLp zz}PllVVm1y=n&A~;;ix$Z2ZRAo#)M5jPuc5&mHuYV^?SaUBv|hVq7rkqC4K6yglHj zF8S!jnMbThsRbH^n6_jT6uU!O6#hXJ08SJZW5?E?;Q6lnel2G%!h>bcAG_wThEp-D z|CM3A$)WQTO!)AgzpgysxK!}R$zN@0d!!>}c-2vAKy2zI5)j}AoHy(2UH4k7eMnog zcg?E|X~ufd`U5|~nK%CA%jpxh!M5=1r@KCSDJ;+6mE(bgx)PIC9u&ogC3xK&D5InC zq?GzMIsk}sD49%9fu!S_>d<(S%XYe;;ovX+Y-i1HjgF_4KW;U3I;#CWA~Qk%Gk_WZF(@a&C4-aYDP7=es$BC8~>xcQ7YKtz2|n= z@y;#KrWcDS%GyXcK7RS)@4lioO!?Bn>Vy*2H_(x18&1`>GL*}R0zbj%hh7;z{oOrM z!JkGSHuR)5N9oP}6Z|DykF%uHVDN2p%z<+ln6p9j9NvGdZtA3#_#)?~f|p+F2P4%{ z8joX$g5Krzsi{E+D@(-kpr4&sE*yh^(?$2+K4zOq84SN?mH52Xr|9!DJ4n(afSCl^OtIpoj<4l`{OIF!pIE_JN#YXot(Ii5PHVokmP|1rZVz3iV8ExX8~BOq27XTOz%9)U=PN$86!xX z2Zau?1C?#l@t_A|+jLZG@NSXTYLln@_)$LwW7~A3_3^Y0um`Y1N9tZRo-r8PrlWsD zEE$j?qm5a#ZI0$T;K!2M!q_&QlGG3~=tqBo&R6c5^0(XfP6a0&_>FH&KkYcIznv+N z;|2<0c-YfSXb_lCn|+vQ9(hZ&&B=vi=v?(^z@4#;#UL<#6*Rz?gp^L-CeJ1zaT2fB z;dV-*Z_6L%;IT5QO07BS8<7)Bcnt^FYW35?XrKm>&4N=ukVWPewD1eeI3^-h-5?;& zOktK^JbBQEWcmxZ7Jh3}&mu)^{Medp%Ao>phB@-Q)Vc0lCkXt);%B0y4OqY3H zGKO7)wsH?OPRVg8)l%KAQTHPQ@%WUY)c{?)UB~)k=3aTp7I*%3R!WUPp9v6SqKU55 z&DOh)R^y#s-cb1_iF-GuAt;@C+nmf z?`q6!T*JNB%{cCWqh?XZ(>zFbM0p(S46qAlmLPLdq^x%$!U%|0bCMMS&okHAHmDrl zhraYfbyx$qTvrwKC5K6cHFfnP^p#ZN%eoWyQk( za?F2DJ^tn+Hv(Nxlf~w6_F!zA?oEL+twdd8?*c|N*&e96M&L(xOozDT4BKvxXg6O7u=N)y;8G!%w z{KIa$;leXhK8tkj^58CqUWIo&M1Jq2QQ(y$bu3y?#}Va*n79->`IuC2>lvk6!t1Al z+};HO4~A2y+Z;vI+N4dVQOW4>0PM$|^Dg}3zeea`l>r(qX}|&C+BWOJ zm^Q=BiR)n6_9A5e7r#*aL^jhzg!eHdU3~Yl+lVQsRsnHzYN$?sYbJwaQV$Eb8@|z6 zA`j6g+OV@2P!j6XLHM}++{RXz2Q$0XHPWWjraTx9UmN_mI1~ds(B_pSFF(O0{r7nK zjtuNz6VA!b`o?vzVyl^05Xdy6m)av}-|py{tLNn84|>Y-LwzI zS=uCT3`DbBXPnL2ItMJRrsY9-{bUD4vfJ{kPH9firphy{G8CPJ?aR}hxNL%ImykX9 zi7bwY_Yh5TYrVkq*!y$5`q9Jq*kvN;qykf*46dTpfEcX|n1#2r-45+D*>$nRr`_Xz zaBq&g6$Li%c5WjN#Qu2^xGYBTF0LN`t3gXX zo@}mfBfq&iU6!01nrH`LJGJU-Pu$osFQqmS5GNR9hT-TnNt;$iuPJqb!sW5Wh54=y zDES#6;x?YxsSlye!wAn$ zWCehS=I%>BdS$ykcS@-a-EH_y{I7BsL0(;O!cp&^xK}FT*CSGXA~`FgrGRX1>1yQd z$;GT9;64*Tp?=9~(Vo3*GAXn)vl#?1Ub|2|ASUaeJbrp*+>LGbU8pjM*GL7~7`K z1V9AQrrsy>C*P|R$4``R0TRNbv*}hl5v2k3 zV9?!)Q@;51Us&FRgwMG)WJ^OF2~frRt;Ht9pA{BU8dSax#LA6n&_fy z;M6_I&r_s*T((*1u)~io`oj(vwipwiUgMV+;I5G=MG9)Uv&> zmim+IB@Q0H)IS!!TeS)1bzn@J;dD~tEfP46xoy)}_3E{0U~HR=h%`#J!@*K-*{};f z*^$&(*S7XyMgi9|r(Cd6#4rO2FHx7=j86P^QrBXHYcgMld() z1BY&tkARg2;RQWjBTUf4t3%w^aF|1ZLU{w({ICb|0vX=2&@WiHp?lTsFtm+Uha}SG z)KU73Ue{qXmFt1oipIGeNch9kUh+<%H5!|OB~%?^TC5M`3>+lHccYZa!nX0s`@fs4 z*Djg>{S_}L&4gL7aq%8 z9}`omA|S3?b8NL4OKi7$08}*-K28N=tq=b6?#JPeb^{e6*3j<}l?UQNGY-5Ugu~RU zV!Dtgj9#nm;)7FmI^fHvA$P49pZ3N6?2T;uBX`;8^cP^eJ7&|zK3+5e52kt}vq^BC zdGxr(13%hA>qlhba3_55d<0EL@fZN3p4)p(HR1|97>)%~6X^x+f?Mo5QxRF_pcyPi znue-G4bBu~ku?RbpR>?#IF2(OT}`b6pTxu6q=3#^Qu*kVF1gz7epHy}#h%~TdYBTU>C>A|qgBf(ykcgbTnqG2p? zy;HyGhj(6l=hCm}iIg$)s1)@^N>Kh7!%rkb1K=?PKtUNa|t|=~i@-Ad;O=Q!pyC zdYc@hi5l0g3|K<7tpMu^r=>h5Qd1*MTlCcUq!?u|*tKy~!z@k;5o81EHGW>NCVp^~2;e$52SmwvZ^+#dMjgT(PxCiM{2gbH}E{VY{EzzbC z7*=wS4$PAsp3s1a9j1@f=|%Y#2@Y``c@<4ZIya*_vbrO?&G{L=R1eN>=qTcs2H4~ zrr=Xsti!h2YAs4k4XB9U! zY}F)ETYLMg#>kBjxgxm5x&~!F$D3U2W=< z_Ym#BG^EnvBPltxN61N9Agh1ucEL%uJfw~io~r{-LTcl|`2bZvua=y)eIpUN)qB#W zlinVTZIcEG^=HES!UEF!{C>G{O?u1vcXgZ6T?~e8K8u0r8?)uHI^s$?@L+74+M{`Ab(Vc zqDK1={r_3<^@b1nPfI1hr&g0qTE*m>-O!@*ET5b_c>sX zHb(<<{fVrHa53I6_l%p5xN$cK@Pgo32gbH}0y&kjI%rettx@WU&9l z6uw=Jvj=0_e2;DFL`d;AWzDTNAzsKX6-{BtDNzm8ID7E6{f&T%PZNQM2r!6stNF5? z934Cu+a^^spAG^KIbjM2_p9Az42Es4VROu7sdf=f+2J7QLA06B)^7D~ZQ90Q*yis@ zChdxeLPMPY+{9W&W~*`F+ry;IIxq&$%1IClWu%XX_lnlAYE-dGP5YcO4z2P#R@p%A z6)bB<-j8%HpT=pWhET=^X-o;G_S3Q2=ru&Bv~HwEZOo_C{*;I6&yjmo@}N`Ap<}T; zs4Ew?=T#@h=~6-qmk&4o<1rxL6q)8aFot|`OYc3BJ5IZ*C5psIhAjqy(-j|J#&8|2 zJ$T#xP*BSp%N@Mo@@NYi2AtQIHmi4XkkGK2>R%fKb7Z`-w%fr(a|q#U3U?#2y47kS zmFw1`W?*cajcjvhb(?loN}(Ty;Vi0I^ zKa!T8NKHV*jDa!Z)y6HgR2bjceF<2XET1JPJyb8)0NZVA?O?Av*Kd4p z4b9X47E-u=Js8{OABpU$>Ne%F$q}jG=*Rv%zVoOSy>D<_;i;IwawEes7Qld^c(=%P z<5e@=ga>IpL~z%k*HPGxTZJsbYK59-_%4K)egP zxxPH%sz?5_e1aR$rT|qMKHk0uZ`(f`BT?hY#_{$?teOA9*@t5Yra<|v3AQDNFkFUU z1|RRqRIoK|PY$zqPxjK2IOw*;bZ;z+%2CIIv279?HK7`N*&-3$w&dK`ZaU*8UOM*L z^k8h8oFtzj=E7B!Xh9TYe5CiQWRZe_rD(a#7JzgrvJC!Xn%dZ*T0>*grVdlDgw3pi^}9>2+5&-3Qk zH)L0JyywB#p-_qn#p7+7sDWU^=Hmwq>XvlX8iz6_r#y@<-UGGdRSbA{9eEXPM|`J6 zvN^{)5;7pB-U#g)$a^rhO_u~N-lmDDh{FAk1wp<{SsKUFgRyP$gbYE;Cl%DssowSzl}>+3_} z5g(dll|)W})x*OhSGOJP_So}Ci~&e2#hmDl7G4G00e2!b4xdB^|2qDrF_*rK)DYvY z8-L!9e-3t1TM;JSrUzr&Bob=t>Nd?oRl#$Adg4Gu>u73aL`~LGh zd-lk!^LBN0b#--jb@lQ^o-G#jmTTWQ`gdoK2j;RxJ`)yF!!I{K7bM>jL^_qc6&33Jq>N^c)pvHZ}jUaX4QSUtPY z?uHbrlF%6Deyb899J-(K(FY8hL~mkydJwDhdl@+oTt>ONw&`J zdmh?l=S?Vb>4!CFh}pr8$U~QI6ZA%4h6%Qey2!S-+ecrm{`uX*fw?JZ;z@f#W0>oZ zPnz-yOq)AyxAYq?FY$*XygE;2y_i3vW7#}YL!J{-avvgC*R|p_c16JnkO}A|qRqhr z1&75SIgE{StNPOJ>PZtDDjmu*eSx9!$<@y=$e4WEv}cZczb8^`>WkHVF6;ifun^=a zs{->z=IJmgPo8D0s8&K_)#?FKx4FnO*j6)>VZD1pY4;j5#wX7Zy2oUI4}97jS{k=M z_}oY^YtR^Eq+A!X82+pMAFU@S~^i2+WQmp9zg(me-+vE+}H!F=IJQycjSK zDmH0CV|8D>7clDuGt7{s%2=z`C5v`vQ~_~6vT;gZ+T9k#cGsVB zhv^|7K;!%7p2X~-T_BDFmv68fqAyi&Z4p6V+;7Cxgr>B`Dt)o)T#bb{cXUxi>da=> zjqi8c!lR~o=i0OSKI!v($3a{Pz2QFu`a;-m4YsOip>k|1-UfXKS(#Jpfi^dJW1F>~ zx^XeEuB{u&O=t}B1!79tw#Z6~^@pPpE|Pmqv@l>MG=>>OmTC-I_OgfHT4(%D=GzJgi~;G4*3`{mF<8xM6=KKjFi2GTRHM!du~&d3<#A z8Z%-82^@gM;s>34^{e?9vwlOwhhNv{BR?sD#b})fb8sgM{$eiAN4_f4EjpN?#P(Gs zA60IWS9wfmoYsVJT?!*GA>xH!t1o0?Lo&fZ8cgsE+4^$HOO3C{#7Qu)a-1ty$Y7c! ziv&Uj^+of;NA9!!WHg3(HM*wsx7gI4cKf#j7kU?w!wrL4#x4Kd`M!U89^L|dM^B%h z9GeamLtpZd6~d3)a2k!ckW;t0s`1Uzwg_Ec;j-P9`^yE;P}tMT;WVKkCdX5^F&~LA zG?z!FFYRuq7^@&K{mEzy^B!W}OCIW+&tAz#_Q>XImLJ_{n^Ly%_}9*V=!4&Sf5VUI zH-faI+xSL*C3}qztN_26mHEN4yg2N)e`FtYo8-Q|7OPdbOJ~_miL z^ky2aOGf}ujgPF5Y19=T=p0L1)@^az{h!O9IOC{ZI8-((m*irsbsiWUq?FJre)$zJ z<<>xxReSM=aNcY4r}GBx7$Up;1hCFnQn>8ZhtTJ0IEaSG(@Y z*LOl*y>-7l^|{tX&hl#L1>=*!;#vyEPwVZWHNLZQ{DHgsq?}M!O5d1bCn;j(WWB(8D3)wC=VhBD3DjN^S!YPtbWyyO-UvV3j^yXP)VoVti9=2vRpf ziVgI#bOZ6!e|r8k3+-`Es7ehQtIC?hOgjIG+n>DbLC@^2d1b1KC!sOS3#rx{c|N0V zCYd02L~6%#Z$Ej@b|&AGW~@bHnEO*TZU!*aI?N-~FN_?-Ts3OY7$z^AbmP4ZiMooc z_bpY*`plh+e2EOYxSJ~3Eiz=1jTp(K#+A;vZu zKTiZS9;ciqdjbHl=y#Htzz;Z*#Osp1TA~ciTU_*WZ6XeOs?zo-r;bxli%Hx!eG`(30bx z{O^Ha`}KP3dn_)wWFn@qD!HlZiX+ctGcI=StjilF z?u0ceS;yJYi)t%Ma&bt&ZRx8Od0*1zee~o#(*_OmRjEb8s(4IamGykLiCiDsCDdd! z?QTnO0CfzGPC{dt>j2KJE0|Z3aXxGvIA8PNTJ?JrVeCnYr-cowI=_j#W~lqt#2JML zx?9Nb4qJLP zA$^&-gs>CPFsa7z!H3;ToF~Ozsg##c{wu%e)^_A%W<3e?!Zwxx)a<%3fI)}h z>a8n5a`|)ytMG%Ij0fKo+7@92wuMN5)@<*zbFS>Y?0 zqDH5ItgYf9V_+I{V5AKb`dv9e-?#B{b>tOL1REe(QPJG3=bFg74pp1#bT7hSPaDcfl8Gr!pAs@D}QGDrWT$9uTO4Z=k0$y$p$ila!}anjr|L#&-8uw$tz1$tn%@6dDE6F1t+0o~1*0l0)Bu3Jg$|_71o?S`hPsMXUTAbLZu+$yVnhAHFT94u{4NLb~~{I8%e; zPwU_a-h`Is;loC(gQ+_5r8MwTyYjx&vF@yRHVO5pQC4L=1|`ka+;Rlh+=3($2cX?0 z(n@G5OIa!@WlG!Fyr4ZP7C!R*9YpBKMh|W#gk}xk&llm*(Otel?ZSMvPR+Q!K#)^@ zg&_Zjxv6C1i^*uPCGHy6(MFEby84j6y`?w$cRnoe*TQv>C#VDSCUbA7{z&XW{w?eX8x$*Y)-3}LIesr#(%|ZDOq!VIghnyTqacfp z?r>x&&v;PK<#h`|V{7aV;Ia=U?(2PD1cxvZz;6ME8YiDxX`J)8QLHhBKw$YtHs^uM zw>k0HJ+Lr7>CDUL_iyqN`Y$W{I%(!m1H1W4bmS%-+43)yvNgY2tLOVi|F)E-th4+h zlD={MlKZX5Ik-D=bptk@pWgG<%T9cLyOKMw*c1|Gi-d$?j(}2MBvH>G&v>#{Nu}fk zy9kpzBlALiQCr0e>{qg@7)?YoD55n{Z-^-Pyb}G>U9&#&&fMI^t3hL!Tr8X3YxArT zOWuBGWaj=1m`YsS^`3s~#I3twnMlH6_Aj{$2@_iYJqluZJFHmFiQQ;XyPyE(?`no+ z?U=q(IL0bf;M?>^g06>^+;$?Ip!p&b+_UlIi>po>we*I5rNqznpCk+061-^!VxvK`*1coh&@6*EmDq;G#NqY24v}5?y&He;)(mMJFVzdI zBs5k%?g%>ussnQ}o1KsRdZGNGqQQE?(CfDt^5G-9mfZPNsQ5^THKyLhW1RU|&4W4Zh?UxGrns7`g0oLWf}sN@ zG{oE!g7)sQ+eqASE=!gqZUU{V)w-tR#j*nT6zQF2iK$rNVIw zv!Kln=qq^$XPmgA)vOo7=loQWz3(iYy@U_%Cu0R7qq|ap89&goLIu4q6-AIAy|l>0ji)-xtY8W{#h}R9TbMox1#t zSq~00eg%7ZPN@2fYe6!r=5a`1br|;$LhW?L#g<;usB1BM*mTpEbo%rQD-RVd@N4UT z06R7x*3<08>OYf4jgNe;f)G>l^0J~$NP1m0sGLrO0aXBTU5dW6yU)N!#ek|m8I2)- zPL}7eb}_edr~}D!bP=1QRiSn<=Lx;ymxlr$*ID_EvHB($k49%f%yF`48AWF6a55XC zQeF$kF2YAX(nK^?%FZlhpqA2tMY^bvAYg#_Ke}N4&kCtQV}%?_QN4igAgW;BlTOLm z97R>$k4*KbA3JHX+?ZA48!Lm~xEAr50#9ZeA3A@DVSkz33(AuVrv?o$r_cD`UzS5= zXk0Gb+A_tBMXKmLd-OVzI~v(P*2xoJBU4)|)=0hm#E3B4DpMF`Syz9 z-`1cve6HbWz*LbQTM+6PSCO;GEut~Zmw;J01{9Ga%UHRjG!vgzk$gmNrQAo(d0|~e zYg*uXIQx^t?I$$Ew4f=snHaS&wo4jdT-4A#oBz!jugT$NPyaXeuKXIY8uY|x?{Y6i zS{=42L`iU(R$t(BNehOtQyK!vul2=K6U~p(e3c zBkF}I!7j&dbnPC;@o?PR(n6uO35`kfX-dbZi;h?H`uy&^{XSgk&{Fp3h*=YE_;C_2 zlc1~8S?ymev)ZPmY*_O(UGA&H1!u{@DA}b&qr5u$xvndmuusWtA)I~9Qu)XxJh5x( zPiG*Gc$4P`FLmSrdzH`(IGBOpFZ_z6hLQ-z=R1ZV!{VRCRcj!!)#xo^PXt6o=$owh zA&l(1MF%Xp=X|DpPD0-j8r6yNyJV&Z67OQ{Wei4WbM>#y(Z)R^q8CL^BRWsw`imH9 zbU7<}5=b>2R|vmS^|2@fx>VkHg~22?USbHqf`%`+HnH=~RCzuuX_p0H8Ve_og%`X; zBUi<2vcAwt`0?o^&{u{v*80qf;b-nh>Da~YfCUE8VdUp>hod#13^iyBa|SV~N@%oQ zSyevrEFH*aAy$ULeR-pWo~q}*Y$m9+`*&Z!?9{3C*6!O`8%Jv##-+c7Y?27Ls1s;2 z;qpV&yc4~NXbkfW*8DW7d61MgzuobOl)J^M=y+xBeOj!LSVY;E-bVShUlk?lSR2$; zhp1?*nkM=eSX057-Ag%xg#4d*-0m&>H&mh6*D&Jg&}Rbid7$&0&0(UfWJcfaa?U~@ zPdQ68FlTI`z}a>p)=)nwDHMD-d0`-!FQYqs7IJ^3;>#CP*x%e6T`{i}tntQXTJSW%) zzVl%Rfo%D>-Em{6N|%fx6?M4u$6olb%YiICd#YyznaSxr&)LD`W)54w&iR)0oL#!y ztOve0V}0G0<0Y#9UpUh=H+Df#XPi~jvlGc#u4lpIm_H+1>pyvKu;7`e20PLgXo9bJ z7!TsGVf8pH1E_YMpnpjpw4QMmIeuGY8TzalK;3<*VtCCC)cKXsk4I{;q4P)+E~MFN zdj;jXgvCB0#TKpJ{Mg=iV)Y^#s|rK^cs6JZua@Qogf)A5ZuCQ}N7~b;i75VSd@zOY`OIU{(X-7RxqE|a<+62P+`7N+$+?xtYidx?XB1am zo;@kjW)}VIr{|2BvCZ&O_RMyJ5AQOejlQ@q`Buy%)aCJ=g*gQ>JJ37m%pL#TZG%0G zd~)O=1$$y~F>^QG_SWtGgn=mUTLhCOIWdyRZZmt||6Fp;%NrZw^nQVXFxD^9-tn)S z=T1cOm$g6s>6XPN1=B)NxnuA^a|J0$^I}ZW2xL0VzJiBF6|6yTO}BvfSA%ch2h(8I zFgR>UGbA*IxeEHFV=AW#OkJGF9{yp_`ula~YD(_8hjJ4d!`!!*$}OI=NO!DQ<4xwW z35{V+#Pm$NSIBi}TguYROd6kK)}S%W7bDCde~}fU_hq)zHDK1DG0dI%1U`eT3LXVl zhI2k^&=}@{H8A1WtNrK{GomrfWli^TU82D9ls-5w^U%!wZo2a+m;vR}%ahEeG=>@U zIo40xmXb&tPJ5ENCp3l`j6kkS-ey=}Q}@0T=Jmc~kATUaC0I70G0b2o;`2umr~{Wl z@>esbMb2liCJBvUhWS}ucaBvy-^p&=;m#rMVdQbqzQ1W?Tl}!*Wh-%ET}no8_<=sg zw~3W5ix|%VGodleVBg|h4CCRLWyak1!9HAt$>k7iU_xVIC24LG8cG6YKMx&Q*S6Bfm0Ni-Am7DqYGwJucK1v)I{fRHDlzP-DFXPh99aUx zaXfoMPiFjSCO?9%A;Vp0MvKS2ZSKF$-*we#P@hv}v9Sh?VLriHo<{oEgor?JH1IF` zo%>vu9f+AklH1Vn`)3dCpN(!SxgO$-EI~*!Rqi0p96TztR?lB#Pb_)cIWwN?0?ec;xT&xL z3krV+mK&$?O=uKT(#E$4W>5gRm>!GPvld@)@$@(Efga_UHE0a;8hVU}3ucJjkXZwc z&B$GfTLJUfqT-1>v6`sod)>dN7v0BW0N&g_8+ga@Ii^U2(7BRJRSh1lh35`{SH*&diL_9&s z%|Fn*i&Z*8LSvXr9qZl`ixHYgl*iVzxN)9{C!sOSC&=eJMNIJt5J*09;bXu3uH`FW zCRQk+F-(7I@9-j~EZ86t_vEu~J!bfNi=x9O7C50X%nsK5wxT>u<;6inuHDqIBs7Nk zVO4obn?jZJrQLlXtENHb`jgQZ=0`O!p?mTn4rXZ{Y+>KGmNWo-9`S0h#he3 zz$qVKT&;qc&=}@oPNJyMc2 zx~9g4sa$#s9E+WazU!2mW)?ku#nn6KCU_Wviq$$9zBm$hWt+x6S3*-!^7IhY-Qcp; z7MF+Ec3yf=d|L$27fzO+foUio(gU*STaQJ^`oqf2;}HDln8Ursrl)c*-@|mRkU}u z4wth>s+xrr2C*6(!==;?UE>#d4;u$5?`~%q{ezgreAt;KoE&&a2DCsTl8u}Q zblZnPH1B8h-Hye4mFsbeuMFitz#VEIoJ3(EIz~}hwa|kQemJ4wf1<5s_w=Pb~mSah=95rYv>m532TT2rzZ~CRegW0m7OJCS6 z-a@OESHJ_O+dUrIpvEVayo8=HwcL2bFL4qs$&?w)*E*Gvd(6(jC{Z2`NiI?O*A zH+~zD`N+3daF$+>wSSk{semBx}7n0)G8Cf3` z?1$XFu+Aw@ul(S&2fym&2PYaneB{@l`LI4_FYaknt^!vqNyJS> zQ%)cDtDCZ+IrAqELg@mH6>WXtfVvAs#p_>?cM2!`8^BC3O?CN(dI-YF$N$i5Dak3Q zX+@Q4n;5tVW2WE+jGXvi{BKeJ67jfGiwsJGJ*xxcvsFf* z#Au{rXyS#Wl1K|0P~B`tC*a&5AqjmW9vM>7L*+ui${-RYvA!A>N$+jWy#1Uq4xrJ+ae2q6h5rZ(yk zxhskt0k!adxe1$(&Ri5%fdtOv2}KgCkuv4ZFS2XAXk$_6^FUR;6OW3x()fbfL=f%n zBY_T=*d{aveG^FCzeyh(IkB*aZIK~oVIM@}S$Tx?m&ynd)GMJk{BKErmFP?8ts;bI z#A$a|7v-MN7-rxho|Ox$aMwkAT%^3XIMO9FhWSkmJ|iEyQ$8aaVw&&aE|9txwG#Bo zOnd77n?jjdPQ^b}jBrMxa&ni61VgLHFDOYhi=0`noE|((*Env<*orCX6q*(|onvhK zF3gKGIcCU`i5<-e|5tX-CoQU%QJRN_=B43VKq>sC$tut%J5NvF+>&BKop&{ljpem1}GkF%v7MK&UOD`I~(Z?~rDWh5?+_%>suFI6UJ`jbtH4Ri`6G!+kEhC^_+VJaWe zx8Pa?WU(um=EKrZ<;MKSk?uJm39FX2fUjeSn}IJYBq{|I=7k6nT*$pi&i*N!g~bMI zW)UYQ9c~Y4r5ZG>t;bx1z*szBZ4r&Pkbh)5b^G(Ni_JZ>l%09+R=4eY+r_ADetfM4 zjbZML7Sy1VKreY$e}iNINA()C>>d!S_MBe zVnS~M=dqTKp5<159&;R|{wlup4;#wt#i}39)Tzz30UsUkFuHiO4ZK{ie*EP696KH) z5^LZFN^1q4#Q;QSZ_%%|U}LRB=)U!h&b2sPje;v0aN%s63`dfr|)yf1)hvBY-8edvTt3vFqjL#vF(5z~6^QOmI zyBgmbG={kiG3R*hqeMJ*T0qzF+8sA^AdwL8mCzVw3oubb4(?0_Gnu=TasI4i;G=AO z4+`AGB_;G$(tYH9QPGiW!Jy+KKVX1V+^@cPqz{}VpII^fs+PA!Vu4-~{%=;w4tRaw z*=I~-9CY`5H0zgvwO#dE=|~ zQ+Hb)*OyJ&^SS-M-wb?e#|rg|)g+-eeELJ~#-eQHj%p;!UGs#ACk*H^7(U30tXGVg z&=8Z!zueJ$93yHH%*eIF*ZuFUt08^*-A!#z?{@g+knQ`Vi48zPqnPC-(Nyk(BBoqo zg)C{G&AROC3ohHs%i%{*mB)a-qhI|bjzr!#`8>*TS;@Cnmd`DlAIA;+>uoaGKYEY8 z=Zd`;O7_0Uqy~*)o=pDd6#2K6Y<;PMsXv+7H%uNo?^uKhYVzz@(2>B4s2esm7x`qp zbDwW|#QVU^Sz2D&%cixpsXqQTmQC`lmH8;^@ee3vMOhXDFHt>xs>vpyG0eGDG0nf! zmnyQ+pYo-dg}u%23FuxT=2ODwOo(0bwTa<|USIn2e#>q0Jbl(5*e2bg1FQ*+@#*J8 z38vLcUko#}MdFdAs-wFG`RxKfd2iEK#hb`Pbp$dF)JfAlCNx%+?V$%9J+QgdhTHur zn$ix1>IuF^BqiY6rYz} z34OQd?uvxo@TU_06|^(>zm`qRhh4go9W&>(8ynY`8xuYMR}$a*5V3zCekN{rQF_B? zYF|H9WSYydjRoIErb{@v^oMyv9$WuMZMO4;>Kd~@bD6M&hU>@rP>yQ9c+LT~t7;?=Y< zqncV;aSLUgdx{n3!@~aUrQO{t;6V}6KJh@Yd|PhN7k8hKgyy#CqrSNRfB|S|AgN3C z_&*bpA}>*AvWjx3?pRFo<4EROU)f+(q-8%JX=^g=E~$yF#pp}gm{px$uehOO#Fx5^ zI&CIyd4er_^}?pjUu8#fNxc#pt5;CcSV6=H^(|#vP8_iMH=8l5@eg7jgDp#F3^R1F zxI^j*xNPbh^Y5KE^I%{mO_b0W=Dm==`;_e!V~Xw|tb6t{cdY$H6UXB3(OWB;J)tp7 zMoPK)f*EvR&Xqe~Gxt$k1Xy>{Ru^yU*K($G^O!Yg43h~@#W=?Dv?=~F&;RM6xg*$# zw=XJrLPN|2?2j}I9lgtw>zVPgv6Ty^#by|Wq$HP)cz1(sMg#i&>P;2VD724Tbl;2Z zEMg+(s&stLx%Z-&VtX&mAJG`*yA;73@(}YWx8UG%fIYri^YVVrtncllRnI+K6h}gD z`1L%wo$ze-TjpY1wBNhPb3$X72O-clk+-&?-@$graBJ_VnY5;l zG(MYJ8pbgVC@!?FZ$eH`X6*ho90##&^!68j>-n$SaT8en((djoHlF@uG@Xep&R|@S z+g|>%xK7&cn5Oeq_*@-;%tbmO>!vFS^vjFnrP8^BOy zjG>;y2uCu~AnJssq9}JIbvs_YwR=bC0DPzj-{G-N&l33Sm1` zB!P1``qHjZur+8Zi(4xR$V;QhCNv_G{FJ7qYVyM!wqf#HU#zSCWHgn9uldV`ms{n4 z0n2XWFvCY4)l-o#GcP+-TpXcTlky1Rq9#QFd#aCN!!cR;$8hI2QkOwpac-xvxfSVy zd3*jL0TL_#_j7O4)4m%d%le9}iwgUz3pY$~H72747e&*Gyc{U%P>5=*|X6j2&Sob(}VKJkQC*68n73cJP$ za(lX=ozS>>_$R1a5L`l1$>7jmZbLB0tY(y3l_)xyW#VKxg3$gCf^9`y>r81YHz7-R zLsJ<6>n+X)qP>Z#bqvV=tx+qOTqJDi?ep|d6n8q;~>6B z@?8urxfAdy+Anwp*70>eE^R+SfkkD)$t5ou?OkbdR-Och+Hvm?oCU}vw2SVJKgi+J zPJS$N+#2*+^YU55-(T>9O)!-c-si%ZNy5t#8e)Ek%9lUjr12w)A+}QmdR(HFo+?Oa z4Abvvd{N|6mVppu{F^mqFZ9Pg%R(P7q}vjQT_!Zd^auU8pWwIq3yRz}KKLcxIe+2MNoeTS_azRg!gw4IqZ!k|w@jE1@yW z>xp@=Unr9*JOM1DLn%KJGya-hR#^y0~YWj3+MuD_ly(}41zi8y6RPBB(gC< z_pH*%rD{+fVECVr?(ZnG!deOmn#dILBbrW zQ5Xsl1U4dQg4tgliIA)3X^}(V%vxRJ|J@lZDPCArA?DKTR#N7J{%ZF!ftxC1WPXA$ zA8w7a@k5pg8Sj)F&NEEgV<`(~p}Q7r=$BRMX9}gMWK(T16z%k>2nexvz7W6xpNKBctphneVE{%08 z2`yHqb;JfdD9j?{JwgI(Ni!X?a=zN8hgrqz)`mywDUfsN75WeRvP+od1cWipnpTgF+71Vs~g zTV}z!pX$ZW`fY0)?l#LzA8LGS<=(?@X&HU$6wDB<f_FMNW%+)hpNr6N;?-bT6sVDykUNR!x)DhArk^qMFAGk-EJb z$+?qM8o)Fq%a5CeVzmYGyfocI=O4#`f1>o3(Ws!fq$c#>{QQor#Y+PTjaB7bV0QGS zLE(tX^{WP6avsJVX?Ad|e9>nqIOb_ji|DPWUf|rlBk_8L+od8;`1IB7&lCnN@wXH%qmC4 z_TqkTQ?Xc+m#&bQE{3^Aydo$p0N+^1xeRKcTAiB;xjVs}PWOy7Ss0tN!rA^m9d*#` zy}7xOObZG%|7N6H98(N*$SN8w<-^0yC-=c@Tc4FA@GA#9a1Vv^6Fu@c<+yPskUyeL zUL^PNd_r~CPP?No`^wS`xL1X|s5+3Jj*UuA)pM^E$#bh4ZfG%8{SP^Amge-CclpRP zk9npS`C@;WH1Ta-B1@$8(1WP%*-lBM;)|Bt^Fk6@O%w+yWHgnAdr?TLw1||5ll3nP zNzfIQf{DYuS|rhrgP+kMB%$w^c4#MZhZRYzDRo=I1liI9f@J7@MHd$&Yz7K8j%J~u z{>>3$bI2K?u(ot8<{{QZQ8VliX-`PIss2(i5c+AZKezT2^&lPywPKls)%?bXeVht* z%0Ki4PyQ;jw3}kcSa#KIzB_)2{={1PRGT)BIGmq&PUjKlL2P&k~^Fn#9d7g z&9=Qg(zOan=;78Ib%w}hW~IuERCJE?u_d>r(ia@qpcRN^sXtXr6Mb9l!!ACD^f9Ki z6+Haj_np5R!ZKmck`2s+#`s)|n7QzSyJ}`yyB*chn;E*v?gf!%4SM2xp8X{gXNrCW zWhitHX70=P$3u5E9`w!>#|SluDv+3(%H&3(ZoT+VVzmqn(@j%QRW}502C};CN1sXDxac8f&rMVIO1(!6DQM;Qi{MDdQ^ z>O_uVIM?}Ou4Bug&bgCWgT^qQWlyDP0H34TPJCDsMtXa!@EJ5tloErczPK4e5_+KN zt<+ntIeiOMg3D}Y@&sviqT2h1?MyMIdUgAfxM3fHLubDuM8SK=k*g7E6wWajssEoLCU`;BKvS;_K3>Vd)8>m z57~YkP2m2{$6Pi>ED9U&)l2KN&%rtgCNivb?qJ0&;CWpKGGMqrVlms?l(C3OK>5Ww z+WdXCU)Q4VS-!p>_~qe(icwcsi0D~AYLDx|X339U`Tj^_mFW>+yIj%wS7-`dj>~uA zS1Y+Qi^YnV5aq9H!DtGvv;SdJEQ#vzoJW+%)rY`TaBgnLjakpf*J@w6q+xAA=R|me zOWCu}mL7ie^i8X*;~Rp3CMjs^kp-4#`rWDk0gR<`ZTf8284UzQK{r8qHx z6DQ^OD!J38{$cCadV`+XK@iL2|NTr#c@T?Fr6fT_t?M9*92P~#@|u_{Sw-K(v8$|h zAvbqF`h6KMl3Pf>n}{UTMQ24efyBk=C60A*f&=N`qQ0<&NcxKEpSy9te9u1@vtSC1nE z2m&vULweqF+mXNz>#3$0`cft1h_0W6g_I=Yh_LXs+89Z{>>F`yR_h%$P@N>-c^&_d zPoXd+uk%NAzV~(m@)WPr{Yx5@r2eCn-m1uKv+f+Ri=j`LT~x;CSqJ>{NFUDzgv{(x zh@L8r5YK8i6Bo-@aci*9QKxHf;R{nSeQ9@-q+-ElByH#3{RQ4;hYsQ|#5`B(9&Cm{ zVm7OLp{%eHN^kc+AY`II*>*dDu9lnn$YCc*D&4lwipYlW&>8UOekH_Ov#Y%JE32)lqs= zK7aMZV~_!qI|RiTKPp+`dw$-erVR9H6B^pE-!o=PTO$_V%VQZjTLZufB!K%^T0GF} zx|ZF$jlfbGr#Q05%&ZcpqolQoHCv*KDuLXWi2C`z$k|ls=Rp$$37&eqX6qM&cIpfI z7trIuWes|(IY(;up42=jrfN^7<7pL>o6s2MDuCws3SEf&wv6R)M{=s3fkl$k0@n9ug$Z@=EiB05f&YXtJnc|+k8K^*jCy^ww~YFW`wOnVqD9U^@ZjXl!(Nt zGn>`92>lL13-r218mi+?Dym3AW0(&R^9Ix#n9b~LeAo_HHmv!YF89^ps9<_nz^2;g zJ$dZ*4<5aC32o)2ct7r7#X;RMEz|0Khs@`-v00HiV*E1oN~S>Ona67iUU%N-s{4`B zszD5zg6Cpn)w^LcwQK4Kwa|B)~l;y(QGfgs3lOZCD%kwXjfw-ta>*=vc9Y zWw_fz_ugp9V_zSRnkUMd&=}@sSm<)xWQH{TS`8nzK$Ly>x4(89+=XXlXF%a9tfETD z%GK`496B%pY^B^(hbFLQ`ptG-u0Hg!4NM#REQOEyN@!TkM9jFoO<&sGH=<`XXvoek z;K#Ltb6ry?`n!viqK(Jj%c^+jg`yM47baE0Q>=cUOJi0kR?D}>e4Z`(EFKV9mBt(}|FBhi%^8Fa zo-gXwpfSvWRNAB2vaunu0bWBkZP6(-Z9-$1`?D=Nq-leaO4XxoWqj74F-#xUcBx&Xphx}redqkADL+!u@+nQnTqnIku9j*eB|6A!EZ2mAF`FY`JX$XF-&Hx zQ~xUPV55~RMX!shm98t#M0{%x#GJx*US9g-p%!`z&hbAXPX zH=0e#M}8#_uGe|z+-XN{y5BhP`9aa{s$yoJZt+|9#u^8g+&dzkVoRggx;z0)dZnT? z99D!^S^M{Qu5#J7er?a!mQHzETkG!{rC^RkkYGY%wY`X#$4hMkn>y>rPRV+$zwl5b zEK&*9pfOB(C+=Hml|Zk$ojv*Ds?$a-y&*7@=>3GoFrOjj+@i-X{T&O;(|SyMeoiBu zsJx{^xe1M7a<{|1EqsQ~B^rrI;SsOAcFwWmk*8O00j@z~n0^l@?>WKNS!a}H$D*Rk zdrl1+!yJWaz4n~o4%w7aZX(_oeCF44)}S#=E)}>>rS5^=_zZ27%9Nb z4(z8hM6Uv7LSvXK67xi9_n=F5CQ1m_KztLla5Wv1S*YmUT<}eOl|7Ek+@HgdN zP`~fJBG$DrtK!ljm-ulbM5-?4gUXE1f7g%gcDbl^a4Gc2Y&L11{;%~oAnMXJXpHx@ zxpMWNqKws5%_I#A&jJ63^^1p!gvKy8118ot{K_Z~bRgX~GXRF>5m7!Ht6DM=B{T+q z23fvFRIO7?SoK-b%gUj7s)|_(dWhGMc`08-cyHiia#ak|5Mw9M$E|OUUY`$_9zS7o z=I2U`cS2*Bf1?BkuAL;#!a1jq}X7rOqukBOAvmUeRppJq%+o_H+*(3w-A<)A*uoV zmJeIMjN*Zkz%!{0v{IcE*NIX__AIyLWvaaE!cyZdVxx(^sE!>-3(#eJm3|*#rzTy5 z(*t?KXMO(sE@M{oGZL?~yRg>Z7e_p_s{gY^U`o}~??(m(`j{PD2g~z@I3PB;sbOs8 z#o7sgiJ3KU-9^k*G*Z>5-7O$faZggEshDFMQ?5r5Q)Oo}HS5^3$V1dm{@B@8{k9M6`f8G-VTRE|QOKfRTZV`=P{zx7! zVI5;k*|uLWGQ*_;=RfoT(`C1!tt#Z4&=_WjyNWSIZdlgo-SOW19aiTCOVULW8e*P` zlHJ}M1Lu&3xZ6ovBY&<;jS%SX-Vl8V?v72?h2f=3dh|CKCG&DM|DzTA0E+GC$5Y>~ z83zU}UNMBR3JVaezEFM0a`{4-6UTEz5L*p;D{BX|0*-L<3i^t&0-vVZ`eJ2W!m<{T zvTD#;SDkky{7y1rtt!ummBC2cWC^prYZ z`+zA!46c#2=(B#pSs;;O-`3NmlD^#cwbLL=6?z!rbW<`HD0dqZp(iw2mGYVxB|GZ; z4eRK^K|V?4+g*>M3=B;Bx}|2p`!h{kB=6{LZ4Kyb)K0g64THYS0*F8T*>b=tk z4c-Eni;9&E{ZTApbm%tu!KfyI!c{PN)2;lIWQ-lMVNLMUUdH zYa@XQz(%9Fr1$L;?>Z;+h9Br-e2vw~S>6n?&vhhJ%XCEg{E}6qX7ksfBR-h&$VzLL zRBB68YAcRKT3L9fG89Z|Xl_7uKuMnJ)5=xG#%oRUn`Fd6J{5v2f5|HUDC%2>Equ0dm%C17^+_M54?i8>6DlEZC<{>*jm zBnr#KsxPiiY81?;{u0_RcL0KZrX+c=Q9x1s2#S(xK|s-#4Gvu%+f#$YBoQ2ebWT&T zaI{^ErHY)G1#!;lgIomSa+* zAm-V=kFq)zi9W0t+LJn=sl0LfiY)pikvQs9RL35{X? zgI;QHY5vfmRa5g!JiLcCFA* zUpleJHncnxFhg{CRmWG4Rapz}%%(`D8pgBIo7e(hf{MX*SFw}|wp zPSq2Ul%32NqD2rls594w|I8;aFBLPlgvJCim`5@fla5x!jLV0- zMkOEMYtR_xp~UQ8?7TKB7D;Xi%&x_{Cp3onEHTd(k2OeK-Q(;pn;$oOuNyW6X5wom zG={kd7YFiLD_$4!r)_dej!f0F+eU7_(>dJX?PPY1$(vXMaOdT?!9f> z1ZClBct62`YKOV4nHGIPZA3z0z%ez{7uQ=zf*UWtz_-vB*GEW#gP9CJCX=vW5)X4W zRtO~Gx?tG-Zp}fQS|tlC3BBQmHP|>)kr^w&mvl8AzlHeul6qB@0rv!=zD)R%}=}jn&PnB64Cf_?fJ3y2yp5RvN>6gH^e(*wsvx zBv^)t+w--NVpU)PmBug!4FKj@q8deUf?lf5iE}68e?()Lbtu>^Q5^12hW|4wT;=3S zLPJcNJ;!aXbZC-$ld*Pi?}gvL_nR&EM&0wBMF%K9Ey=}*jP8<-Zo5*A$*Lq7Tn8l5 zDnTN;sAT7b(;a{FDPH^Fu`f>?GRzNf-exaOn|eu6uT{3WK(I@K${q*4%ZxO1Khb%N zw|f$L6;63s;{QvW&A_q=Q(vlt3F%L^v|*S$+@~Qu!52_}&~iky>p)+F#xS=9Cc=Ig z7SlRq4E8u+i^F#|SJIbum!Ev7L1WkyLLm;t$jQ;Cr8^U!ylHL7&O4W!jPTV~(6GJKI>|zzxKB zWMBM!g+7~qb4V#W`=J$%`KkS!(yY9k5&W(II$3h3n zwE7(bcXd1~3=s)TCVlzh+fyjeS>2g*CKINVx?(*>L=!)-!in0pd)De;+t^}Y4D zwp+^Atys!V{?nW*{`J&>kV|r~BB3$NJ-8ZtiSRjreTxrsV6x|%mOtg;A9zsZX&H}d z&=}^H#C%pddZ5>+r0j<6N88o8nQKd{@+TF@Ut<|55 zhP4jGul_5Fhe@g|x9;^*SHJh-!*OkL{z50u+a&fmw`!3GB{O>Ra2baeA6%5gKFg?v z6q4crGfw#;={aF|NoW*v-gcDj9JW;KItV>7M1-8%$&i)MD5g&?tM|?a9@Gno?hH;a z6PgP7yMK!&2TX~Z!`;c2_e*8vv)sNydml%+B=pAf!<55kMSNR)(*QlToo|cP(QFz% z5co;xiT@n>iu(f_@CtDaf|Fy`KwsKB%fK3~k1A$)HsCt?W8}m&uM!;AX=?h4b%QI( zM%4+Yx`S54Kgjm;(%A#skd^<-=PgKQm$IGE80I*7A1?@IFm&caDi4NPgT^qoC+1Z} z@tQj60*)EaP?v;;m_E1dU8sV)PYiOPSN8I{b1mWy>25V>4AW;qx?M0sy=-2mixS3X z4I0B-4O;H*`~^O-ey9USr+mt0(>sK_f_10sSz$tT>nt)0+p!&EwVm{^)y)uKR^te&^-<3Ykshyw2`> z9@=H+O-k90rGI_$^7=S7IZG6={0&6bF<@*<%VFili4A!qRi0fel&8mjvhNlhu;`xi zw<)<+uveiSGDjRxvLTJ~yOFJOHAt1EN68)^+0x(L_s*>CFx~jw#^buq8*>8WH>W5Z zb2?S&!SxVolh0!3YZ>ePgag?2@DB&QwfnbC%WYnAgM`ar)O|i|b2`&g?T#5ED=LC{ zO6z}DMn$$5`qO8p_MTF59~SE>!#&QNuW;gwqk0W$@*T|g>)WHc7P|#zy*mA^(y#R= zqoMpfCU0@URd3DXThL|w!VVvn2GL&H-BHjD)Fl*Lmmg-gjg7R+CM4Ew+psQgLuzZ( zmG?l?_(%5mW!Wb?jo7u6?f3LQ2T$5(PqbUvi?Dnn)Zahy)bKmt+Q(8Q6tQhOD^SzqW1$<9&*4FTz-+l=YcA7kOXzuhRB< z6JPe%E$hL+_IG4TEI!mpNf4)hffQ$b+d$ICeY+k0!=Pvsr=oaJxO^}mc&Q) z&EXqtx9>4999pD%Kv2TK2@d_AIY8PD<%3z$VV?O&Cq>Z*ma+wh4qoP-*VX~^e-?oR zy-jF{>FwK>DBevF%wR>u*X&-(bpZ>T>F|}%80PoHd<>OjW34}OmbzonTVK*aSQie>l(@A}9J**20X5;7og5@* z-aO+RIHJI~e$w1{ME!)uhH;IK*=vF(Nx_H%*Ow}-u0I(K@%Lm0`XdC{(T%xr09xGN z#Ok9i4r76ZDja9s5&t}*G0+m#bB?Hd*f-a8-YE51Gn+A<9|@zFg01k?D{HQG*9tV; zFaeG?28*BEgGt2Zp5y~oU!Z717CoaB-Ki2tXdmV zOY&+39wbs@@lrPZu?NPz`|!TrBYX%p$9+!O*40mFYnzNp*7(9o*o#fVtqnBBv8*v4 z*#%#BufJ$#IPNE2veG^SpILfK`d1GE-2EG*9es~zC?g?U^EjkcZgULu4|5q^AIsjK z{`$@*-L=q``0`SGH1BQzV|!>^3qC@o(N^Yr{R2zqBfEI9)*W7&I=qzifAg<5J?Hvw zNe;gzL)%iWxeUT+X{#R6AUuS0jac0pG{o$|BLBw6WmJWD>%!JGcldJs)8C&t6_{s< zx{M|xe1v-?mBuh{pm=U9VwyjE?j4`6K6DvkURuOVXsmT;!8*FP;u!rDcEO-lQtR0o z^uk$sqE-nFVNU>Jc?njZ_p1OiEc3}A^`cI8xGmgIL|TK!Fb5O!u_C7QE*Puz4nFmz z6&iRzGQmt}hW^EF&z{?C5N36=H@@tJL5J1P` z;TEs9@}$O=DQLU$!VtHAy@&u-FLWR-{XCqn;^?1X?xHA7Qmi&6CNFX99p!Y)z96S<2taEe5l2hm!a@qMwt~=P|qfE zab6-#jG53FCS4169s5k|Ez6h%UqF~)#@Nm+LH&>w?p@@9RZFSlZ2Lwc`d0E?&$r>$ zEcSBIqfqtNrnHP6bNr5_?7Wp8e|h)@a1c_=gvP4!AThsT>%}dsfq&U1&)u~36D49M zCL*CB=667H+w(C;F#QTAANf7JOO~i9wyFFh`CleaO zWSVI=P5MrdgE%KR4h#GDQ)H;)lTr#FF?T*VH70stV z8BHjH%~38kpSg=V52%8f&=B(#vi&L_uaSqi@0ky#`|wrzb3#MR#n@VYk9Tpr0W%Dk zx=5Ve_H&n)|9tRXzVCT@ZgzGo>L(Gn2LSVdhH;QcjW4YvHq7+in>S&!9NRK!GIwLc zXX5Z;dZ;f|1~>i5Xbjr#!0trdk1@@Qy6(6YUYh+DD&cqmMFqPF4Ke9SxC5zovxj7WngZZtvX=|0-@nXV?BoMPKGxYRhg=haY>gycoW`SO&YKREa7ppFF z<*I*BgWjqb=;J1`si08meK^eDZ0|`sKbwUQax2W!^*7 z%FbI}=?#B(Knr~=b{olpE4fQin-Uts{3m%}vbu`iCepwODxHesa1S*)ZoY(um_Jf< zU8vEwNDnBQU|?!~hIaQy_Q-ftHcX{4%%zF>6VSnD!0fZX8+687+mx~^e|U4Vi92(I z%RRXoG=@2bn74Oo)iQJ%+UDAE@7Yi+Ml{f2@|2!@oiMJ8+oX`{hcO;zzIbgMlK3Iix zt#jy0yW38*AaJ2S+2*aG4hE@PTD0JJFz@)Vo6@pzr8BPE{(+Sti4O(yT(ZJP4u!j7 zr3Wlua`S{F*aEQ~2%gcFuwu_944q0;B|zHz?D!k5-?#qYA=QdaXbkgyVm=_4fd@Nk z6>^s(d?qx+WFz_echp7)aoQ53iAU+5=SOqML)@qdz2OJ?xWj0F&W=8Bv!<1Y^VDAA z#wIj|xd#nU;>Jqg2R4oGyWpO2%dP@NOgl_M1I+2;{`ar_o8p>qJJ;}a@}*5)_2$=VZAwp)pLq%l(mbsi51^1-C0@N8NtS#@j7-uwl+?rq1kzM$4XmbIOSKuE2iV zkh!f#uh4hT64>-J4Nl12He!o{Y9HQm%-^;-0k-AFSL>(lw!EYy_tn~zkelHnbgw`! zi%KhF_B-yU=Qb*D3KnwvR)dDs@*DXK*qI*PJycgd2m^zESYl9n!LFV+2IW8CEC0y! z79M+XB{JB}twLt`$SibHUsyP|S$sT%TayB)i?(Id{m#UBMzBY^b(7E-CW9Z`+C|K@ z8YZ22`Fv~?Uw_N}OMLlHO!wTdqTsi>YvJBseYsL8zb$6F(yXqo7tqT7)WaMwjA{*I zwfL}`5whW@_WR`9b=IvCKVgS=q~*8vUZSu0A`p`M3y0u%A;Tt_sxnCVC(;Z{sR@lW zrFVYw?X>;LCLh`I>rHRIYE7(6Nq6Dmh%{mpb|4c=#JxEkYER^ra1&tCBHAf)5X>B}3VxjGir zNQViSyD)iCs88~xm2@ym@=6w$B*zf2Al>`E^LIOY(k5uV-=QKEmk{$?>id6^L|ClgvK!a&9r}$ zF(I_^oGW*}X6~aKm$D!49ohQw^}Np^!Axij^FGSq@*5jZ+j6oYolvzS!nEv?Z{Y72UygPXtA)N*+4F@H+CFV|Y#KSd?D?9AB)VDP_ge+9F zU-G?`_dew^KxF*}0r|)+F%H@MY00B5Ie*F1e0lgGx17t;5_a>XozNTpi^RXF)3%g# ziX|bVRRuHXrM0Ef7^a`(uSWWK(@r)+V0W%OtbWUN-VmOBI0C~g+;Hj7VG%*}FA6wU z<2@4Gc64h)rdibS309hqe3cB#LAsid#+a!`pH2S9t*amR)M+L6IBOo~LY3YbtpHiA zSgbohH1uw^1Rt5MDCA*9NNMEyp~-<+gT^q&5pyxp*G2RaKL!tHz4q6pb}UV6w0q~d|7D;4VPfAc=>;u~wk+RH7mtIYjX{n>*+0fI9ll?J(&EFiTjjSU zY*j@Rdi#1NdDHjsZc#rXcn^0d-oW-|{0BZPt}}b%iyz9@Y>EVsXtAJ_6GbdRfozb~ zW6BlehY5L@7l<2zNKW1KLyZIa--_fmG)iF8%7J@aJ&5dY!hL|(3mo$tS}Q>4zZI3N zLBobz2u%1?Asvg3lYP~CaVTPESr5fz8V-yL#iU7tJZy$AZ>z1=IPmV(IRP00HY5Ka zw)y5^>OPp;TlfvWuQ+T-^C^J<{k;r4zdrnci%=CN+@`f^`>VuhCNw64UBPRGt2gg> zO8O9RWf>^!g}Pyqa|a9OB<+NrrJPSVI+S@dTA7(V{3Gk})J^Sc_D4*P%sA*i=b^DR zXbkfL0{s@4P#zlznvkf|;8fjMydD?%S*ujbR=|K94R+Sc2s? z!$h>#pk)tPj9vR?F+o8O5*lJ|0q)B4h&hP#@qi|aGCeiKc_1)bSeLlF+g^gG#G6Qf z#4#+kDEf=7eb4IZmZD$|phf95@rbti%r!LA=oXDM&ME66mpcR0AUTe#@?oW$ zmdbI-sH0DwRMgw5DlA7QhTJ&<9gG+N{9kVJB@?BWly`fU5J~L`8h`x47RxfVuvNlw zWJLqpu(}@Ob`|CHU4{>fX3UFS$agDW?2@e4NYFr$xC!nb*;iluapggeZClFz{_lOC z+2OOp&_EIeREx$MeF!mcrFp9>_a!DD+066aU-g0q5Qp6V>Z7(=q}POiS%bzfxjNxy zPy_0`WJEmAZ9Zf4F+c8J%9gsR?~J?tatbi>x(5*^G{p29mpc-3Cp;>YQ_S#crEJxM zhkrNzQr4YuFlgzZ@Cl7!b_J>HM*3LcZQwZIlxzOdcElt<+aM3+-XP_XisoiGT4kK4 z^@AT)yei5|zPIwurmEa6atZw1-PHE3MSFs6%lb zY!-0oxm$g@^3FWGTrWddC^w-o%)yk)^+mbJ76vBsvf9eqH4|AUG{mItx&?g9!2^5- zT^Ak@3nhzy@Nbl!5pTF1ozb`B8ppOE^cF&{@u4#B_9OHA3-e)YwSq1=WXxJGtV7Fq zmzby;G{(F?&~}d~U*JJkBcjyVKMH0I8pB+J0{Ou!L)1~b<_YsMIn;nd7`Z^o)YFb0 zxiHq~7IjuR6}Dz6`~IMX=kGRa&yq{nkm?uwPTwv*vanI~C$Oz2L9?0ff4u$=*E3X) z)18VYNNB8TK6`nCh$}1x=_0*dd)4lKW;1njex;U=+$!R_3-Q9bmE7CKV%7gPN8E~C zKT@%v=5n7L7GB$)xo6uakJ}Yh$uVot80KjD4BHmFmw9u;Umt|S4m$yJcCmXUG=|Bn z{)MgkqURmWOK^Hhq*Q~(p#KI^_Xz1@YoJ>hxub@FMwNUJK|*7geuHusY0W?{KHT6^ z_U_VG%>C&|E;uK#r+N) z@BWeP_R&|Xe||SSF_{+bXT?VSL7lYhd_RBP&x#F0yC~6Yoy|o+HIDF?xvrf2=irH0^DD;_x#xM^g<{fyT zrGq#%5Fwff7rcF0k{Okx-l8pG@$pZ7=`1&y@e6FVlKVkR`iWW+{!39y)M!|UEu zCd(1nwN!dPM*1l4cuPK^C;q#H?un-D7~mR_PIzi#eQPVOu)%2(I1y;SFa%k5(3dJZ zdHRzb{j)kS{MKzH?H9Jb?YvmaHs9QE*lkS4nnb!LG{(1|5WOeuHe~8Z>L&JJ!`BKlJ62ZU2NuF=ZBem-MlKDY2ldl(HVP|98*lo;pdAL2`n^tZ^Wsff zCClf}1N@YR_~qr;l@AwH2wmA$7IY<_-Okgm#wO`mZng6^H9n$*>?3zraS%8EMIsoO zP%yDU2@Ns*Lb})Ue}EhG-(ar~8+yi^VW$r(Wy2>vu;a+78v%16RxXbJPiTmFKD$IC zA9L^kW)QZp*C(EUC%fPYjq$k_ig#;^fCHbluBa1>Dtsn1hWRx4T)2qYGG_Z-S8K*m zsfT~w_tvqiFgh&Bn;?q5dMSJRo-x}_Yu~cu#DuOhn7P7TPoT?=I&CK2OI}yJ zcs%=ZWH!=ook-^{BLn%9BeMyOiQG?1-WHJuv+={E-OhRS)4saiGXR*$Y$2gB%yTK8 zycvUfTK0km$4>hsyM@=8Cw^!`W0>Bh&W{Uoy}ryxc6HBLufMh*j%@$!)6GtP{1~nw z<#pek{fdwL!1HfPnXZHC($&pqz%n0M{miKohi$YymZBEF=EEiK+0%>M9}n}(uRTS} zN3&XdD;g}9}(q&Q%qTr&WppKtzhy%kn0Y5D`NIDBM}p1s}?Yp(>KYm-rj zKYr;C`}$U9RmzvJK4XhjkqHc1d9B-8FRAO;1;r*iSqY6+rTj23I~KuRC9b};sXP`b z>|!y=fb*GNDe)OvdsWoE!Vn}hhW#b^?N;PhE>It$yX0$t-$v1(Af|-IF#7<`jVFD~ zr=8W3foLmWCN?dhG0b{kb}Yg)pm_o|UK?jf5#xz^UVY>1jAn=uKEVQvLm;SMTlwt2lW%x?qCYlP3BDG7~X?n}&DXoTu~ z)(=Vjaq?pWwz}*T4BaCRxMumQpW_{EdFGdp9web5<`CLF?^Rz+D-dg+rr@-wr1rGQ z5*lKzfZCMTrlnX>S^iZ3Nn5Ih_E18O1Y(ejm#o)2n zlK@mcm4oETQk5mx27H*4p83$jLssOGHZdYKXo%^XaWHX~lB!(GcRnoDMzk=I7*Y)? z%g?G|xhRiB365D|79>}u`{CI8xl0#|wPjzMzbj^F6^roDV>)hKF>j$*nUCP`QoKs# z127(;1ec|B;&fk@?PAtCJ8ib%g}g8?&qfp!ETN$UJ*GElw?Zv|Ne93`GCT11-p&K= zxZ=#?fSG$10duKOPM&ne`q=+_;^C%+`#gDUNv&#d2k*ajy#u$~2McJQJoxl3mma>O zA5na<+}&`CNU=dWqIy4!%yenrimgFoRbhOJ+rMc2%~r{7FOK49)t^W49gFKKiN>^4 zM$aW*TWS2-xngOuQi$W(zuB(K)rUTY)RB2<lF%!D6apGsd|B6q-ywK(zhaOhTW_I1;#XB01a%%(8w~(bm;{33i96v;;6{6` zwESsz{=eR?Jz z^?KB+*CioJTf`$(u8Mjzb*XwkT6J(qt8sJDh9Vlu{eJiQt+n_0?K4OE`RJWLetvS+ zZ|$|$Yp=cb-fQy^Y7&r}4s)B=h@+mxmQiWeAAsPaXnUQGcy#i;KRJ5~U(SO17I8oJ z_iAB98LF|XGTmsCz*tw9rgj5JLyYs!kM;i5-RB-wC@$J#!1q=)vnvUE5*Wkz1L9-` zx(cTb9mRd;-*fVn+bpZa@&v|k{){;9$m7(lPT8L4t!wo?fiawq5ohL#LiRvBCib=H zI=*TDLx8U|%k|74@ah<{9M}j@okb^gV2G0mqt$zJh*&`ziE|#d zw5Bg*H(QJ(;MAY8q?zmrCeA1c72{O*o(-#vQ~Gl4OjlZbPA9;a4&D&BD6*A9R3T`Yg5 zbS5x{)8}h9X>Mt$S*$5p1FeBOfic|QCcTMgk&p#Wa@+Ak@9KXHJe%rnFvK~O zYQKQN!*RlcFg`TsL3gc%Qg)XL&N?uL(}!RGDREH9oJf?2J6^PdOxXHmDHiLg^zJGZ znfIvkfi>X6^kaqS`D6+o*<$RVaAJ&Z=3lAbcc7=0zO4&7u|1WCo4Jq6KQe!T6F@qH zc7vp!fegz5sLNMY1Q6R;q>->uI*ct1$D_);guS@l`H7V$ywyB``QGUCMX_~sqHYL5~)97B(e&^2SVwHh(p%~|fHLNvB*MVX8 zJY)N@dn=@SK{JdU`eK6cyFopC9E~VgM)S*ZZ6YZ3f^!SR;{~DVUcan34vr5hJ%{3> zN&^b8ma4^^d|^G{7p^QVoXOts5h{5u-t@vtj(Z)IlE8i6V_*Dcv+1okD&HE_&Z_gN zHFX}F%5Dq|@HR8jSt%-QlAKr<5*Wita@-kt@|M?% z!pAh*S)t)8X5--1v*1HQa{^;Hxhv336W)eC+6?jZNh^QW-3gpf4bog{#G}&Jwa&!c zfz?Foe4~6RHQ*cfb8(gJWIi9Ztt9dfNu4q?bafQ&exivB&522)OouKxvsueJySdV} zkfpYjCiPD!k0v6h17kRSDZ#-_;*+9Oc3kdWbSK*LU@bUyo{y~a5sbAkjWAv*Vw}Q= zc()1!!d{kqdAv=Vcv*GRJXE^tm=E@ZhLR7hp=a1ow#su2-}wj|gLH+x>{bcO>%bV! z#{r6x%dCE^#WK2ynh#rZwE~Sa1TQHbpstKW5sKC}tgUw7e0EFg0^AwW*(sjC4i0PD zAHx2&NISsWxTe$VvfBb04;IydF`Ryx;A+qgg9|J*#N>1`bp< zlY4^kPpHa<743LrPzudbt)n~A2L6SL79X9mtCRXyUR!h6c<%q4Oa{k_lQ1tm!XY5> zk%>$MAwscJLvprkVgU|<##v%s84L6?P$jGhT}RVAy&*?9q&tX2^~IepRv*kSq0V~D zDyHrNG=UypfKEweRM~lo>&e<@FF{~D;y{k+UcELYbBFk9y-o8ShH{iFo7vqy0~Glc zcjB<>>-KqQj!mqf1CDjaysN&HwF^PN^e6LjhLa?@i^YS4F5E6F`KaP~P+`h#?1H;U z*b*jVRxoA7^&foj?Gpzdm^LfZ7wo9sW66J?e`QBMpMUk7m`l!OWtzjzZ@X$iDxmX> zanh_NCW_;BzWlCni}os{&FX5DRheOLzsB+1yYei55Zt;Y^<}omsx+B$)qx?-dnpjV zf&4zaK`NmumMYbC%(8n@d~h8Y;v7!S+)1fqlFs`=l)VwB_)a5-e8T(1fyyAC8er9F z#NxhIaXe1zH}vY+~{N zLy2{>|7qTNMq5XB%LJqm+d8pJmri*KWLz1A;}_V`GPl&;iZ@^7mUrsS#&IC9E(9G3 zyXk&>U2?AHRfeQ!z|}gRQ(8a3mu?A^vNF60)fG|rPv&_LT`Tu}f`GtL$jSdzMW#DY zO)nybQ5aJ?yW8+3Mnop&xSz7Cqub4>KE|7%QH3o{riqNe1I-3alzt=TR-u>OCE~N{ zz!*+0LAgr_ALA4Yz}l-86RB}rIe{^pvx#%_Je?NDhI*ITQSCry0z;hMWbOr9+;pKc zu-xF!o7yn(sXL#B2TK$^fl-`(e01LzoWZ}_q4rP4jz4nqlLq_h^L{*Kim0-QKOL2s zZfcfD_r80?gHvCB}u5!PsZ|@SqJwm*0P>;4Xx&RJjowOvb&t z`+Lj3v&Q2E%F?OCJJO%lYRAd>rcP{(p-Odjw0Bjjl&Ax{qnzmMPWw;dO*voDw=sPsWmVw^??nZaq=3hRNhhmK!+&!2p-ZftXl z^P)2+`}mD58{F`R#~`}I0VOadIv!NX9p?#Jp?=XEB(|@x`B+*BjG?Bu+%-aO5R^p{ zr!3v7&vFQ=oW^PijNyEOIOzpoWg)=QJVf!GzP+Bj_q%I=bG-QAIxvQlI}j>^XLn2& zmt?*qc1dc!#ph@3^VFodJTrAdo)Za-wQvUrc2o0$lYt#|hBw%9(AKy9VgtlT$q<#m z7|wU;3Wv!(3|-xGeK~{fPOnWd;i9o?8jHXD-jrLub2l4F2v1}jteE1ZSZ#^hp z1YodMD(y0UE+n^g+PLZyl&cp_zgMSpWu2-I7-4W{% z&!M>BgwBQYW_7ghzRO+{=0eORL|`*oySgw{o{+sVVDnqT2hQkrj@OK3KY-2?NHZUH zCy>3A-Sg~A?|Lxo`)!b{zhTC`A!-wFnjh7d8aVZ*xOC=eZaw;rUBq1zI&s@fbJxz8 zUZDKSglcwlA>F=nXY-6X7=F7ZWSItd9eoIzl;tRt|G90as zN`Nchr$+dq+8$**!ifT#mip-3eGa_y=YMdX2Zc{dh(>Fu5z+$te|hFFZh03zGF45X zHG+Y7)}8VH#>%Y7WmnZq;RoOYP8Bf7k~tvqJT2_nA!Iin77W-6{p0;~b!{*{9uqvJ z8&P@H_bERkt95&evk6kH$@1H&xIDF)ovaUcTGg@ka(XkIAdlLoRe0n+64=yFfHUTx zF-r_zf+jJXmXT^I`;aARn3F)z#sAGKYt1BcIF>qjWh zOK+7Ah=m4};J^P1UNzI*(lRmzQzLOFz8Wu;vUVJspdkxPgzgUZBzB59csN>Zc05Hs z*qW^wH78$NYg}h@Uy#-UpUp1zeERk4&L^MInhD7Lr|`KBJn?%2J)HQ*=9wr8fUuzd zy&rGe2&UuUQC@Zl3~`Pkx@HcKx8bddLy1yRH#|H`D&fK`t|fNy$x;wOpviaEj$aYj zRcH^hYf%^!oRqZauAqIn&|U|IICBmzmagfkcG{#{V22Zjn7|wU4H?h%n+cEDH4FP? zVy_{%BQI)94}wgX0UnZWU)%>?NsCgdajqXD_X7`gpa>zA8MD5qz)Cwtovs6~oh}6? z#Y~k{AnttEwecDQKmubpj{psozNL2AkP_ofcvN=x)>ht}hx3g52JsFEgDT6ve2SE~ zdRSj&m4G^ew2qxj0ydkza2~QY64`k@sbdiTQ2$N=dUr7?j(KJdMP4?B(cG2iSps7u z)3dst6F$Z%a(GJPTV67r;7ni)=W61-FOSm*46N?}1L?XU_h}&()7}Fz7ECCO;O$@r8-D za{jY}zV-pxn)&u%SqTiuy%AXD1d{jF@l4v_^#yKML=~oX`jaI-l?9P2jAxf3p|y=s zEu3WkM*?Fw_owPVj0Z#&+^a6KNBR2usdn)!IDs*oKB3Fox_~qA$`Y%OY&mV!e&naB z^{;^1@$OdOuLE!Rz1!TZX;yO=Zd1)NgjG$6#H&WPY}OdiCoqPd7YDmbg#5e~RpDGB zB-ep4oJ?xEYY88FElUKB8gbpNcOrpVV--GuAx@wFe}*_S6$_Rw@VJ>n_lk2Q_3%eh zFFkzh84`Hnzlx~2+cw27d^1mrOakyTQ{lp|hn+1b%GJR* z0+UYF)H!4~5ARw~l6e(xrFV8yYq$s1(!NKGUbxlmlc)H&-t**(Bs}4vtvG7wsIq%V zWF7dNz*xWhIjbwAUjbensg+tOyI;ahR(a_J#&8DN#cM;hDMoL~DZ7XBbS5w+Uoy-6 zTIdYC60_d0P`q{fqrdUx=cj=G`v}fDFvOYL?-}n8)tpx5(m3LehOE)|yxYSr7fsx{ z8}c9~c6pH#d%oGvQGLb|KK8bv+F0(9EXk&}#nbsc)Xa!ik&wV!w*q{OU&ui8KY7nL z*FLx}YZoLpAsfxO`}XUT{%~@^^^s=74BjO4o3-a3JgfH{ubu4HYL?5Y49sTmT}@px zIy(K~_prk>87vU@h*s5sv2NXnM%+!LC+?Qf(~zAWsL?I6Y9&CM4c#(jRyJv~QmEy! ze6zM@bmxa(`_^Cfx*y#9rYK%ek_5(ZHnJ;ciZTayktoW_o_PLQ-J=UH){>C)0E##C z+3y_x_rc2xx^FvlMS6=A`xPUnja|I_IWHmaBbU%eUIr%u4hH*>;0~n-rHu35Xf|6D ztOLWYJiuP~b91bDP$d=fVK2XN{e3&VH4iu+$+IVcAI*K?%hRbp8I0llD{xl&%xPWNgq{Df zQpzV$5Qjx2z*do(z!+MdOmXw`+M$YBJml$KW6v4R?k6}C7{kf)8}76`PAfUH{mO$r z_sa_?rmgZUO<)YCKON=$Qrve-@}4_((a~d;66gJSoCyqZ?o0^{1*lec2-`1(SMf0J zb`pNpfiawZS@~JQ$8J91KkiwnjWd#?;q<1F;1uh`=01I?fm44n7{lo|hJAK8e@K4* zp1PE&eW>Yf%Shs)_J@qF5E<2hA)Umz`X_jBk8cUf3r*nZA)G+*VcUD`g;DcFkl^p( zix#?2#*ORIthGFoMi$-oJNp)PCSqg*7?Zl?Dt`U#Lswl~ zm|ReBpJ~a!gG|kF=*|c?uC_?A_gzk1ZC-HO=0OaLh-a7|QE3XL&r|g?^UAxTHM*U9 zWH7HfFe>TNB;q_#ls>>~3Z|%KTFUNOGBe)BpTHRN12bdo6ds~N*6$vk^vcV>IRC{fd?xM;7{%ZP7QHM@E((aM~6FJtb~@r~^ZsbeAs9 z3kP`dK2TMCG55-xuX>X@@Wk(pD_BcS7E9il2ih}z2sn+!3S)eD4?7Ul76QE4j=q%L zHa*&x!Whnvh;w_&qQRF_k(w&f{b=8G*>y;T(xk1v=K!awv~CRUwu+1dhn+ zOAW;Olff9`(Lk&WD1aWvhb6kQ`zKjZ`UEkuIxvQFYvRlfh2@P5=S6g1nhjNTU<_v` z888IPv^FdpwD72AzmYO8H$=uVO1up_PMp1nD}u2m?qm}qs+;&q2Uh8&?rZ3|xB~P< zU+Dk6#fF~8E>1PyOhCceDffbSntU&!z|kmtityJ_YZ_b`3g#b0t3s|dwaV^CdCju@ z>AFeL-6Qf0X(6E!WlZ$QKAZK1?awRr_t(gD&Y96QvC74X|Hit{1t$;q%Bb0_l$h{5 zfgx(zbcs`S88Bm;7QcleFk0ujhD|3hMlN?XVB2kT7ixaZYr^s4xy`8Qwg7i;S376X zgeXv^7_GzZ<=S$I;H||a=!)Bo9wP3A;K$iwaX%=@m*Fol^;sD>9Y;u7aG;^JeTG}u zS$&sA@-5r)F&H;2sKK(ns^cSDbVI2$DSd|`JvxGgu*J${hym_caQ}f%;edi>TPP

#05~@<;nf=y*G5jHHZ;+I;k76IVbi^)8m*Jj5IsJnQ zD^M|dv1tAkN8S<~pkAEZIJr+D1CQd#*y-7HS;R7=ptH3=v3h9>sFmWz%Vf%F7Err1u$$c;;=-IPV2fdtjbtQ^!x>r@D6<4%d%AoVY* zAO{O+AEi4<@d|JcsuS$V7QE%dF)+(4kf;M=TzE6$LV&k*D-3Ob~ zW%{BP6-KkC&H3y%V|6ZU?rNVnfhYK|nXMg%-kUK8=Z3axsB77grXXaE4x%t2i^c1} zfcAC8e=hCwnu|C~PP#D&TX+*#Zn3j27S|ZF5*Uhl2M*-i+*j#`HYT-U+^QNz@QeQ| z>I;Vv{}FTMI4F<9CD}pwIQCa)`%K$feGY!AY>c)J^+O{X^EamOVW+(91vkXYI2h*d z>N{=1;(jL}ff?fKqMTAXg%t~mMpISMpy>hP|I_+y?={Ld%Anb>dwV>qYLFJ$p` z5SYs;g_N`5{7W{`&1-ndd1A8M?r{@5a442OBvNe4#pVuls{7M zP4^76=(eF6HMmDOdFCS<2eC3Jvft2i0m9C9;@^eWs`;X+roMn_uc2R&i;73DgyMhB zsl3?Zw_}RdLf;R^S?Wcr9gGQ<&Z-Vtcq|Wge+pNn*TMgo7u6RNtUqwqcRCJ79@|{3 zGGgg^k7IxvRwI55JMVLo8)!St4VWR|l=qoOtNW?6Jv z2gY#jNo&k5BhUN(s=jBJk=KDyoLr1+b8RWpxjZFW8A zCTP;{t-am(eYIvM$q8|Z(ZS!cdx;dr+MmE!3x0#-aM%tM@XO7eq)%zBQgj)dU)D74 z>p>P_^hsc>6~8fOO^|7w#tW*OZLMBqE8)5*XtAF))>UzThWp-4CnQrRdM>roew+gWH05O!klI5)|NGDj1feWXp}YUo65QpJeI*X zWH+hRNZN~8jNIT1sd(wzFg<)4b&Q~dw3}?_F9elGZXBuzk0=xmnD)8d{xqo*bSC*p&0b~Om`*HcH13s$kK)q(&ssL+ z=6wsrZC?4&bKCv#_=4Jt=eXyx)K}iEP~7j8qc8aPcHHckEX7J(hn2%k4!myO>Za*F zzvdajl%QKlLB|Y{`~kMPJTUf~AD7qMCn-eGw{l}I=iwQmaR(eOB*Tr)Egf^YMFQOk-X{5$-LY@oN1{{# z-UOkyELdCqyjfp1lW!yW#@gUF?qg{q@JKu$B)NFUS9TwAE%V*UIFrB-=gs7iSFG(s zaRc4rS#a}$zLeeH^NMS9>Wf=SR>v`%G#jOht=azU$}J+A&=pIkSj}(uLB81p#&8aU zj#oCt9_?1YmQb8goe+`xLgp2U`a%u%Ja_@!_a|;pGNpP7rl9tXHO6n;23X##a#EV` zDlTssJ?@2Z*gSw{lL1LOfP~tiIYN6bm77P$ofOU8DuAF zjeLOYq1mvI@=)Z0S&Hr}wq||uZGfE7X@eoKUS#ZrO_B7ahWF8*vf%Vp?^=AP_F~Ui zIK2v|)V#ty1bRnmIi)y(A;so?+~3HA*j*V7n$<45-KpfUZ%SYc=QpUGdx}v6?X(eM z?4bWSvU2RbwG1tRA^NJ^Zk6;vM=|J{4HfZ zq3t@4aRUz?QThXH9q|6_!)1iOTtwQjXaQwgjv&98~vtX4!t}arI zFWMCc5v4w?6L(=Wtn|ep?j-qT+|-w{>z$`MA;UF%I!GS@nT3!2O-`md_M+6Sz6dZ} zW70E!xGj29;njwL_WLPt(j7{K(@) z9b}hkU1Sf~lbu&(X*PmUmdL#05%EBQA!-zsnkl<(4&(7@wgkp-UJWI1*GOEH*L={R z+%CJ5dZbff6lZBEwej z?}_sZc{(lLGdGyZD1jl)oDnw2U1ijDi)!tnGP+i;C;ea%ZeuZvyiBoSO_X6xW9m0; zE-pOVsE|5E^@W+Imfek>O21vOFI$bzFoIJ5SJW53+DRt)+;irMf(BEV%JJzl!MP@gpzN-e z0V1@U_|~*Gxw;qgGsdG{Fl4ooFdXV6J3Cy%gkvK$G@cYDFox3)?EB_5TDr1KyMl@u z7&3MPROxuq@FuOl?to;?x&H8epWcA5$7hN_H}W-k!N|&bb1H<8*%z!+|->0p4dTSe#`}t2| z9d=_rC0<+RM`QYq61TKD*4p?Car#m2U)BTXmFrLE3eve#9%rH)D65dinZPJZOMaBw zIgc~-2`SD5#&G_bbnf!$II~f)bR`t7GA7hgJ+;(2zsZ;4t6fgD=azXztedd@Enm)- zx6ieV8)=)`T8qsy7B+RA()N#%a0pRu$+VAaZ*FViEC7Y8+Gm!E-A-;-h!r;x42OQN zluQ&>wg{vdMnP8Pk`pW@iWA9@B-m2hE9%9AHGE?NV>lNOXA&6MP!@SOx0?I@|%V~EL7{ke%jVpb;JQ&jD(Zc_!EPF&7 z!5G?OY3u^v*-_JTQSLb2x8wkHf}vLUuL(fe%X-*$bKP;kyZwB|by-ap`=Nnsh%D zXA$Nksv4#vZqq!K=%xQNL%T7X+_sX)Y;v2&7+d8wr7x@s(yWDt;0#>m)c67{hrO9rX%u5xO2|&;ip9 zrCHkGeyWRyk5f0QRYgsl>CIVz`Id_lJKET5tj|RFIu6{*2NDheCAspiA`*s&2N5RL zQjtE?PhhZB=-@nv{t`eAtqqcMG@i1}0qXW3eCoWOimq-_wR;huxVQqBgy6ay-dMz# zEMH_>`;*AG#95WwQ<_`CcRuXCttffAlJbiApSbnZoNPORnBKdMZj_kQV6vPK8(FL* zgZ9hM`92R#L8C#U=Io?LYq)sf+4gtResf;0g+`o67VuXa)%rpP%v4`}qpE3XAENvs z9#uLbm?FuQ-QI*j#Rzta5Az=OVmOITr#gWNZyq)O>IF~WwA0fUojYaKs;O}Ei507s zXAXK&!6g=_MFff+2d}+(&K*bA$kFP4hvL>7ZFcS9kM9yJf`@)lbRXMJEvV0?ndp9S z>J3?tV$n*uP9j)=t@~Z<{&kd6oC%U^i*T%Y*~d>d-*-Ao;@={Y41!N!3@7(JyB`V8 z;7xRAQ1P~H7vB1pU;POVmQ0)lKN)JA4ES|l^lw}XS@N(c6?apmU zlnq{!MV)O|mR|a3+7YCkz*tv;&d2#qn}XbP<^HXEp7f~S(Bm)m;|T~{NY;Yd8BHAy zn|exqe@EzH@~Jfy>?QWWVlUWRamtnhUfsE~U#)u^%wYEZU`5MKuYOQ)OGQ{AEV7Y& zY2$HIfBK1(e)eYS!=lXf2YWFx^`*x}w#e?%7)CAF0s^;%h&-ro;sB{aGp_G8OSRa>SX)Bs zq>JPu8|5($>wv7WTJQmaCbDm9c1JdwX=m`vGG=1$si4F*M>kTmvkr{moJ5=%s{)x? zVyAfS4J)o$`8ZhRR$@G_vUZEOg-@R}y6*`GLr5WWDUf#6C&!K1=b6KTCWVW1*1bZcd!v6`Y}8;P2K*rx)!^V5V_t1MW)_^OqC zPXBl9_k(?GNO-N5d22AIGY&XcI8*^D9ZO~xE-7<^*o38WNsCi9~N#&DL1Gg}G`bP7Dq U?#RR*^)ORe#Jo-5cir{>0JvlS4FCWD literal 0 HcmV?d00001 diff --git a/packages/pandas-gbq/prof/test_upload_chinese_unicode_data.prof b/packages/pandas-gbq/prof/test_upload_chinese_unicode_data.prof new file mode 100644 index 0000000000000000000000000000000000000000..593d226a84c4df06c17059a7d789036b1cff7114 GIT binary patch literal 206840 zcmce92b`R>(Z0bxmvgtVu}w487_iTUUT!g%-i-;J_1@jy>({>9U9@|~=IDeLLJ6UV z5JJfp14)2{gb;cPf$$MRAcQ3ZQxd+V8ZZzHA^+#myprB`wYQx4bJ^d|vTk=IjYgx< zXfzsqI<^1mwN9(_!)NtoyLM(v3-fK0r?h2T^KIEFlk@G9r?f09WsAtREXuZ)ip^6S z!iTo**H0(Gs0};UzjJ&(Wi#EVZu_Pqidj``DxQ=nmYSv)mSuBGnwH?3FrOjaDyWb8rAh~v(2EjuotaiOp8s0Os*|coK`Gl zS~{{-V_FPL*=dDL>*7q1Ej9)Dd;pM4S7~7rdfib#uZdI#a40t>$iEhGTbH<9^!D91 zeEhqInr#NSE$TDe@&84%swn9G4mC`DEVpq0U0u=`k)Afnz z{MSaESlsIQW}BHYwZGCG0Z;c^y@{Pzi%>;R(`o_=gxi8EgZBqKa`2stc25Pz?6>s; z-(Pt_s#`Upo~Z*ghX|YeYT2!O=FZ(Z6`0KO{))pI&gV30OcyilS>T#!!&F)yU+E-x z>)?~FU;5#&m~yv)2<0IF7gh#U z?gtWTGb9l)hNNr;!%!70w~i%!z>J_VAL2kLp51(^nuOcXW2%7w3k!D7Sgs7^hOyUk z*=ymym4^RTYW~}~7u`-OR4tiecKdBPhw`YkJQ}^!>p{!*+Zq9++v}mxWJk8J9pUOA zeF(C-tjd(~3V&L~umY*uPwnsefq5zG#mp4vQEM;yv4(m@p`$AZmNm6zIyxea;wpBy zG2PkKQ40Arn3~E&l}2&fF;5>hqP>4A*;B{VnL@$#wIkG$!*(-VA9|Wjf(OoBanCou zvwzBF*v%b#(aq#=wdM=UI9N{;_Yvu~Y_`C!(rg)%f{L1&kHjiMHiJ2I7=EaH#*#zR zi%~M2=?Ir_g_+J`N3oa)@Ar-He!p2AfUf~U?L-|-5d700o0-~wb*2{`S;O&4L+OX| z)YO>?bLeq6mQ!s^rmYRCN+GXGP(WHs*e4`L&BF~u?gKr-;MS230<1F}kW20K|by!L{n_g(*DinNZP;2 zrF+dqN)f6-mWoXH7mnG`AX~z)sW8``2W=?6)nX)8Kd>2KIS4<5yHY0`0!d^fP(`Dd zVA2a&%W6j{cIbV{X5ebOtOAyoLLB%YT_RL8MT6(e@G%59>!8K&}!r}moUCei> zR;XO*C6Fn3g^e%AS6D2(p41$k&J+*F+MNpylqpYjUzJFjTAQGxgej_4vl9ziPV}>B zev5yk$PBhNq$3ZN3X59?BgchbE$yKtwFEzeZ*kU*!RQ)KYr66E8e0FRrq%h zL?^+`vuC`2*A`gLO-5~qB8lD8>)Q_f*)BIvNChS{HQIB%s`ndq<5c+zN~$xE1fN{@ ztK*vvS}SEUV6+{2!DvZQQ7m?#ii zT3b`L*86@Lgmwi~$XZu(vNfi|Tv$rCg{|Cnk%4>UVs##V4JS`Pso~kB(2-h8DBPA^ z+Pi|=!eW9IZ(+VO55^2nrpzNB#+mUBqetCAnb!xV12KRIA)vcwRW2hN(9N-Cmxofh zPq*kzRSoPCh7ENwsquXl8!5_*6b##!p{Ubf0Hr^cks)4n(Q#u@bVM4I1rYXl)u}7O64ZU-yAM$*Aq=);u&ntuknP z(o@8$KNcoTi}6J2IwB?O^^0b`ei7WDw!jaS-3UIokSQ%pqehgO!ZJ4?iR5loJx_QptU34)z;J!2B5E6mN1d6B97~%RsN)OsjJWt87mhRsU%e>qSP%l@l{*m zYw!*YRI-QnR#`PogH#q0ps|Zi7le3w5*VA>dJ*nC_kQpMWj3)i}jxQ zPHe?sxY=Xbp?R1QwK`a#MM>RC9BZwJonQUPL^#qA-rCfXRZU`F%&d~+7+YIR49VfSC3|G2IArlQWDn(i0e2}Dbtx&C?Z7|Gofr`iLc`hkXCmn)Lrwg+fK&!pt9C({GwLD%ae`X- zJF#|&w7LzD<>4wDKu63fEPeUKiMs7^g|Oa`s@?5myN@sABO{U5nb~MUJxD}7#fA2o z7W`7l5{&_ct|D}g*vjOg4dsaNY98T-rO{~3?a5=0zO(k`4IecJCWCU9vz-rhJ5W25 z7Tg=RSbfUnqhD_hOeQ&M&eqEKkcKYmtX~9Y*00DIgGD%Q-(Z4FLt)T(&b!MCz1F zBC@gNlOlHHwYfgV(VLE(J3f46cq%X%o&m1;ft8$+fu+OjqKp=@pYmv7I97Omm(G+* zJSeNzaIzcg?@9?!nil3JN5`7(m+9%ud9X2&0fROmrIVol74N+} zd-^sho570P2X>;!=4j?PXFUg_Jb2fROlM1524sneMf{>ac3B1EthucyvPm_?8|9|G zJPg!45b~nqU~I$C@B(}(Myjv>^2I-#ss^M2lK}-+;MdBh1bG}u0$dEW*r=Keh`0Ew zU;e8sB9WW~4a3K1ni$ZIEgmBsBDo0F$jU|9-Q#xfn1DvHC2Z<(78|0va%rue*T65iTI)h@pg+mU5?c#= zReTv5LdDg~MY7_vh|RjhW`y4SLkB39XbV8eG3*BuYk9L7uv}jZWy}hY=?L;FEG_KR z$FKjS(C!?b-{TTi6FY7eiDVvBFj|+%cr%D2-)iX;@6d zvII5%3kUZjj=?mrAU5jkp=PAk3=CplRP;-%in7ne>6G?4Al83_XG(efbV22pE`!TT zAp>>3Gt-(cE=?Dg6>;R<4m6y;>8)EJaUX2x>>0eQ)_N~~J_uDFO%g>*xN#APp;u#P z-WRND%mpA%POJrme4Zw?@If-cApD-kflq>R!kh`g(-+KXP$NQ}nH-GwdcEI?#9J2% zTzdv5Svc5Pmd?SNgtMKKHIKRx{x7)?7(5*bNxn&%e9?CxpA$kxJGW{&gy|4KtH;nY zv-~)4e5pV2WwH;>3oV~=^F%IHccap$H7ccy~d_sf6R2FK%@rskIf69Ud7qvP%o0qE*+~UOZVdwZFO% zWx@-9b9s=~-nogZjAm^c;mgQ=@Qm>qdIm>kQe-V+Jc{v8pk8d#B49XAYF6hn2Wlc{IjK*mw50jOnx<0_X_ZTY@4}_q!3NwM$ESe7>lQ zeTZb_@(6F;C1ESJ*sa=R!r65#$Oa8glazi}FgHPog0{}n-g=DNw?w7;XscRQC^hl0 z9^S)gIE)eBUl8Akf^M=mJ>eYVeymKlqkMP+Pw`;8q1%X_ZM)-JHH)*YmK6U9%iKoR z8?GDXki(0C8f5*cR_L>-R&|V^{c_23R}jmgZP{Wg{KLN3zXGCUJCu;l4bt*T8;f1H z&~^QZE!R@yk%%7fc5dr{&LVr0UKIQ*3l7FvS*G{+y~8q%OF}q4n2lL4S{g?IIaZhI zf)92_HpsLtOBYW}x(fB!!^>EFGnzBI=+i}^j?E;b8kU1JPc-7jW`~X^bHR0Yp%Lw& zqqap)z&F*qznJ!^?gTJEjxuGe^NGzsis$}YthFfLLdF`)5g3`qUs_o=II8KpwHJC}!Lm4d{Fz=oEKa_+>qhHfF{rn6iFmj`UPDtnL|d*G|cZt!GE5x_5Ki6zhZ_F+*C-0kI)9;9J3o zLhTJK(I2f7?8pA}h}5107lx^9&+Z@Kz!P4?(*4um;enk5rhZoo6gvpFBpKWa>5<$Z z_)AtXu9PpKtaeXj4LnH7eu0;Tso*hIFeY9>ONevZUY7nROApao?DRsKdSdbw{nwwE zAX@!jldoFna2k9_78^{PXh)W8>8+5-)kb^89#7sHXdG$kP5q=H)%lfzqoe&{ZF{P0 zwYc93E`4CXKi{zM^Jbf&qhcQji=;yYoH^k31UKhEyA1pbm_Q4L+fGLZPP6IaSVL4i z<&I($WdtTg8w#r9$>n13UfpanN!mCgvpwSS4BdyKo{&kt7Zr_sK<#iCvxr+b29%)v-&YQotU-wTYs!r0l~DsM>R#wl^UIBuLZ6( zsHK?X2Hb6AYJuye>e~d5bO8tMtprJ?cL#@HdRilUkOpqM>9yb83koC~b%%s7P4Rjl zZVFaUwaTy2kSc&!fjJwr8BVpMzItV)7%rteo=e#}n7b}wml#=@3Nayrk$n+(hR2fj zGpk3|Q(pl7M}pr$8~2zcG>-6?Gz*`rWi-=a3VoB^K8f9yk;55`$xMw(uVU$SBEzWc zdfn`IwmlEoX!ggZpf|OX)k*N&xY>`qaQ5Vs&2XUB{JaW1vwm6W zOpCU+@5t@dK)F?cAt8Z;N}j6$XiW1?~Ds(Y*?F0z1Yenx3nSF>O_|qG3CM zz0BY^2n{)3z&85}2pO>lZcNTrcL4=x;+AO-a+TMn)VEQo?*!GY&c%;F%Pk-HTYq8$ z+ZU{A4hTs0rec;}nR$K5H~a*cLete>0ka&i(TR_JPt*{hQJWg!H&DG!f+dTu8TRN0 zQ&ToW;k8>YR+*%at3IoQHl)u7C`rmAqPt)xDP9cXyy3BdPW(=&vw#pl<`aa84*~(F zWzgR!g9b#3aJBr2=`iR#NRv)#)L0z)P)DaBqLa7T;i}daW}0vbkY^#(DNbY; zCt~NDg~cbUWoStIk6X9v^brDh3-)m#D;cfNjrHb{)}u@(H56AE9HT$6`KqR}@CGcr zDNcKFq|VI*PTWt{*0OYfJyr>KNMOm0tY_8C6x)FMj={BtBt&DC&4EZH`YUHJe=EL? z9Q6jKYz8ApaP66(XpBdH=n*Myk?S`8b;RCpeb5}3i~+wJkY-_V3cR;qD-sPgIuK7G z_H}$&o|2&(`ruM^!uwVqT!2+77o+dk=<0#8>`{S(em})|WiMjcj9M&RbW9*a5)%Qc zqc%u8K-!3lMqzW(|?<5 zBL8moC6A}|_G07gGn9|ZmzmtgU)nlOC%I*3>-<@5CyedQrXQjZfF|2Ex7r=lT{{&o z&^?Sh=pOKDzZ#TIT&MJ4lBmqKs93XHFK~!$9b{zSV{9^XNxcSGRA%WxkGZ#EM4j?r zz-_rCUfWx|;_7y9P@~~ZtZ&!6zwr+z-QSoBOor~yec;3vts@AlR8n27r7j3hE;YNb z|MWNKjR5Xq-KIwPBeqN%n*9uA)T6@vj#eIPCuy~vrB%$3&>K?GZY#!H*#LfQJoi6t z?pBvnV102a6&az|eHQKyUf2PJFo&6%2n$9h=vc)0wtR0c1*Wh)l(p z08IAUaY%d%(wlBO%>LeQ_Dc^RGAtFCOt*Smx?Kk*iN;oz_@%sw$Az`GGuxS0%XpHB z#lX69$F_c+7Qq!5KHmMpp96#P0ErJ&uRs!yjVvB8n#cl#u}Cz&Bbked?QURV?r>(T zdHAgKN!iR|m2F2uWcYPZH7wS=!W415w)HUaE!N=pKa)eoGx-icF!oR5Kk@tbcunz! z4M1oWmrGoB_aVGHCFC?US8TuQVP z21ZYd=JF8jE+NJ|&*~QIV3KBz_v|84O{_XL4-#t*$WWoeg4;;?nz9M%5}N=*DvY4X zB$03`u)^G1(!-!Rj+6x`cJq{YXqKd*j)FV86!)bmxa$;3;xOG#LVe{pikKm@5tKbS zi!e6HIq8_l+c|L|)jQK&iO&dxC{Va^?6#$KiYBpgA}TBvyIKSV$f(*FZ3FVg-Lhf= zxLf@NsDL1KV3JL59wP|2%9zcH9mpqSlj9Fh1MBupydF%l)vax-=`=d5-Pf|kbXo?2 zjv$G)-SB#LQ1HZ;hzTH8bv2CqJldfEKlIkcRSe~|nC!_8-c6~rJ`qfFgU60Ui_7Fp zY4VJvGiS`)v3=VaEi)Jj@@fo~PJ;fMUAz0}3y;F(naW-vpvaf};P^)3?d0x#Iem8V*sSZCqakOu+=-0grbI$`*s zlNQZS1tx>qwZ-b20LK-%V#qUlqJ>tR#ZJaPCW}yv_XR2^5s?oxQk6l>e}uem;7bzN zNezj8A}j*Z#5p8<5gS3B1nDc6ESkQYmVSwlW9|67mkLLo zg&7>Vb-`~afcXukvryq+Nhd+mq4O53b>L1Zn*rT8N+Ba1l^b-hPn?v@#8;{xz6!QG zY1M$XQZ7Z5xCHF#)zHf~1Mfr`@IoKhj_q~?D=8E) z#Jxz{aiSfdJPP;v+3PfJ)?|vUVTkU#&4#xXPz=QY5IUes063ZGBU6#@<;aw}QBOOM z?lF6bF>Sh~!&`ts=jm*N^0?}H7QHYe41(uo5=Bbzw3V2F4jjJVtJkU+5D|x|JpQBp zv2O*c}Vq>oBUy084z9*j z^G{(%C)gN#nC|qtXY4SX;3W?(zhIqqRvx0$2-naaP>s!J2nq&I3=ibev5s70gQx=+GI$-;BCI3_u-go;8=U{={eFJ$bqIW8 zGDe`U#X%p2Vg1B;4r3=eFMFo{dt0$*41BwCgOUpyHaJYKDlT+6JqUzI<^2 zSDCTtut4!0x%q2N4Q%Eb7BL9hmdT(`lhN5QJrLdA7`X(vx1R?1rzGK95VqxkVPO`o zSfYe8dl*{9i8hBM)p01IF>@k~T5(Vnrc-=6OQ zO5aGrGR8FAh!748M}ly0Xob6d2U2~;QGKouBT8~vaO$b*SwzNG1sUg9v_7h*ONh@Y zkf7y(Km_6DFf7gn;T1_@;QTgSfb41!eH?f$WDd*~UHX&X+qDFfrX)_pHgjxInH4nzv8iWw{;WWzLp`yQOFAfmKt!&2DZ>``-!Z-skZdbt#2RF4MXRm7NR4;8Y zjv;mQ%-j@Gu{>0-JcpZAyiU0!Do*w?_C{^f#Mi(AA(P62z{cneMgO@`|6nkyra zA|X0nT1egTpPVg?LI~W*qIT~HDRV68Lc0P5!m~;L!GJH$5353-c-v$Us)c2qW0?^$ zX72HgmoFd3B81_WA{pvCF+F_$gj9u2U38g_*gAd`>QDcsRkOVtDnS7OhtlVHHB zE1sVDdYG~qgSz^=MxElRkXc5DF^rEJR$<}TVGvL!&xb#U#iw^W5p8PRieoi67V^ch zfN!Mo_^%cgTDCsPYWo9uEUrHt2eg<3PZs(&r@o1Ou*oPo)uUf-NMv!1`AbZVr1g8O zaS+L&1F(ln(NCQOhp*i-X8LQ`+nNkWfgk&Yo3Kbj3u4xQf13X_GdNW82$DfIrg=tX4)@puafzz?I~WDcQP6X5%|1D59JeH&% zYWy0pnMp9O48V%m1tib=W9iaAp81c#=$gp@SNii*{(sE)zz~3dfhbLO`R2?+} z6(m4B{MF{dkZz*YVwO3i(Hlf$xE%ULVLTIcOTnj1C!gZoQ3_tULKU4sIL7zQgS%){ z!m-%@=p+idg@r_-Z*7}WM^$6%cj$@gK6?i@SZw<;z((XDp*>3;Bh7}k!lN`g6iiU8 zmSTMuN>mn;-g22WcAMAhYf=!q9cTx{uX)0xS`xch-OHN}*k!w`umd!iZhtql)%O4F z`pw3GF&R~?9s#ctNvz!Rpn6n|Dq0ujJ>kgBZmjXQ8>?5ggYk>{3C6WNm{!2(KKud_ ziHVCDYAm)>zl@JW6xGToRNQ-EN!0p+%rt#Bgh{lki#cT``|L&{{+LYlNPLW5g-oLA zK;;IlVPG$=7-9vyQZ0F6Dx5!9()%4vWc3}ZLYF#qS=CoG6mN&?#E(Qx_md5FmGkJy zIQGM>E!b?Qr(5#*5;~YEG>4aQpa!A|nEB8Jw_JFf0_m;SyH{bET=ozcbo)V`HY9oq(u#4(5kes}p)ftdMX);@uCi;$4gGlqXW* z(WV|og3vxL&aU8E_O}Y2w(c`dw0O8VRsKUXAMXyYuYaO5%4Jw)+TVV3|3nZVr4K(`@m5@ z_G|>Qn5_Cr;`8L9xDx5jha~H1Rdq8OImhZd99oru0o^(h3jK5L{qsK3PdUhMn6CaBSH*4WC ziVD1mQdqLRHc+6G&Kl`41Zdjv)gO)~W{}&9@x@Rpw%E+bRtd=E1~+f82u zBqPyx!Yh<&5j|b&6obUo3d4^n7{(Q#Q$wZd$P^H1h)=`iF|LUUyj4iOEhDf$(Pv`g zkJ^m!-*V&VSvi3Q*NMsvsGvN8dL3-Os#V{*MEj1f%~?ZWGMK}ACgyOjUgx4;OR-6N zw?q$hoe#-eo~*x-E2wyflG;CxmLtC^dA}=0Oqr>%)uV6ZiHYhcvG{C=P#W@&Be@!I zMY@B5ea^vO62wM`le~=C2fC2K%!FRlN&)2@Lrs*2=|2&W8kftsxpQ3Tt@~83R8M)L=$QqJf>C(fwK1|Yfspu}Txqd8zopV-c~9^X=*OP7aTwo>#2*7<6beVz(l5?S1paRd^psJIv^=E%`Q1O}C_a?$j8PpMYK!%F*%&@4X;5YwWNX+bv3N zhsG)|liWulS7Q`mmS~8cqXf8)d2}i$Z@{Z-2e^^Vy&v@4GCuFI6|pFz?#MO{#X?#h ziuKDo95F~mi^Rvx>Q5+&4mc5D`)jVRf_3y(X#`ovQkeH&5)`&on@qANoAx|`7>AU5 z8uD9OQQAbp zZ}WIKFx1P4GeTMpG#Mb4PAD6w2**{%FSFP*d;0d%XTkkw3Eq3x!9J$J0}&k6lyp|b zBWu}d1?|?!cztj=wgaxO&XtE~k7LT`8gn{kB1sZ{lb#8WC72KTfb!wnm_9lQo*yvb z%G67E^yPtZ7YteSue~u*-w-hZ!=-p^`&axFub{y0r6(%i+_3)QHiP-m+zT=#X~Lxd zwsTsDp+Wo(jhIt6r~R4Du-z4-r9SkImXOCp=UOt$fc9!d2H)zBoh*>CZ;`wga{(Cx zdx%NLSFEg$$&(pPUSo7Aa3(9{3`sN#^ zYz8B&FLI4#*ZRX0m|!O}Kz5`IINb-U4i212Mo0jy!(=eN=b@^Qw=+)y2%X2f6bOBmj4-lW~-adrYkjpm2X8h^xL}1Q-#-@`sXcQy_bY znB5%o>w`x~5|+uxsty8H4#Gf|N8<)DxD|IZ=_?pTwQ?w$Xd5={1{n5wuxW)Vd3jt! zeK~*A{(cyo1a;)hF?Kqp1`7eNBO5WX52x7-+W0Zr5Fs@VBFEWG(`^VXUeJE~b@@=u zLCuw2*~W)5tK4CXckGAu2p$C+kP}6`jRBn`FwsNocj z_P<%f2(ez6gRaCdo6VXH4?h(&Mai(xfRaOHp-E#CP)i$0%0q5$!+TK{!pF9#FL9y8 z^*N$Uth|nP;;%IKS}DfGR-+WR0lSb%57)0TUdSH2$Z0$Z*J1I} zg1ib%A8n|@zacJL;n(1yZ;zjT+a0(Mo4Tm`gY~x9Ii*fUClUlwhfUk$8;=c51ttRm zy@(3J8SLp$F!@5%icoWOc&)$mY0XK;HNZXwhAi-dz9}qIuWs4CxT9k(6&4qVg@VXt z^*%c_2zn5L=+qLfu@P6Vf(q#K0#cJnw4Z?W$e5A9g)8tw_!vi`p}j+%#3ig0kTpCO zXxbDwfy~!PgU+7V<7!;2ISeDsm_ zedvH<^ey!FR-(_ux<=THnvAaj;uI$T&}I`25|Yn!5}mBUsb9p~raU_A zT$1k{NP6p4RsV)Q)LD!FYX}OST=I?p)ex|{n2yk^9E&ehGv}~1o{RZ$0O+T*C!Drw z!xU503XnVwU(B?Vt?Qf@qcqj#ie7di&qk+pdK)73zHr5S;=`ntV}mH-yYPO{aCpGr zaz5i@BKkj2skRCwlTThR%sjQVnf#5d7ax~1$D6RYss4CwSZfBld} z$HNk!umQ#5#K(w>^rgTn#ps?>w~XO68ZcU5`+OBXDHG~z;k@7lmlXb~U(gEkX}k$5 z$CXYUF{V^&Q&qo0577yyR>^#hc{r(OATbjlo}2-tv>o&xl&ypHUJhDiqKs!{Rt&$9 z05J#f_9PzF&H;L1g?$@{K>zaEh{M%Wm!?J~v_Tg3iUM8yprkGA$}PtEA#H0^OpfBP zGKt1|rP=PSZ1!D>d+M#CGbNO0x!Ok!Q^L49eyDW`BZ~d|Mo+LYO&@(%SwEu^PqJuh#bY{E?Bg*qcSfQUs)$lB0yjGn=x;Wg1 zL@6JeQip1g*t=JP4km}ms8dlKIO{k?OrnFSxUi)f0C{i>{!@Wl{9`-uWVbf2(=E7L zz}OF-VfPN34%Fu%3AX*B3Bnf7(304xvYTeZQYEianD{xt&5!(bk4w+qBc*1_B*?KA zCbd+d)jRe=AS6#3vT_0xt_MQF^qIzcXvuhRRs%E{^(b0Lm+Po`>JvR;F;(4@UW#71 zsKqTJwPZ&Vu&Nx4_9FecGo$odt8OL1Z~EgneYsTZ{P}#fa=U5}X?ulPi&yXf`=6*c zq_tj0H{_#ZOG1RJP2e0g?&g^(*Mt`2$}k@Hh2qx&J6)ca%be2Zkp#@eotD~zLop2M zcqe>05I94>pE`oSFtKZ_&8UoY$ANguYcD()4d#Ewo=jzcLZYL!1T<~m5;b|0vxk~% zoZ-=^LMM@D=PbxH?dv%!g^upc?(oJx9*2`h-L!#8u1BRK?;;*F>C+}_yRjKmF$lj_ zI+0*qY0YOD&PKvDhnuo#Ceg))Elh;1EmG=p;mloFH4{rkn*kwMbvOg&ELf)E{Li-K_Aq#60lerhqp#>O{1U z;j6PQtCd2BJ@B8rVCX5dpo2m^p}u(*EqTwdcTB9Sg3TE4ud~KqOYQRs&cxZJq(9F$ zt2gmAaJ+>eOst2T&6uEwb$V(V<*S5HP_@V8!TFw|y2tD*rr`QKnUJt4`ymz|5W0>C zF{!aacHwJ)xU~v@>yMQ_*mf=Mg$!Opr3^9}+NXZTn9~G*nmYuv(v=%jW=WQ~JGK#? zh@Fu@TU_%vQj}dfu$np4r`DUn*p>&RTRW4dlRl$v0@45}Czfh$XS41jq2{P(ZJupi#%%zzT- z!65A^z%i#Zp7GwW{b?Ivx$q)Vh&pl+XD?Ko>+=kndzzxOg~is@+O}9?qR>e2#L>Hb zH@ceVki^(ds@R({YrW3QvKHDv5Zjlw0v@xJ5TzjMfSZ)oekLl>2zEeY@;|I*yi`Ls zuy`%#i?hIaqQ~@QI}A*(@3x@X&p-gBLo5!7&WUwQ%<8Fc?Kdm<;yB;*s?dcGPf>x` zdiDc7zECdDlpDz#y(cz`9n^(%OCwMS5B_EY+L!eanPXKe4UJ*il;pjQ@^F+iR4tv1 zCb<#Si4}*z{&(zn-6ziuGfjqV04a}*6^w?XM-!D+LWpfRgqtU}iOYf=#z+?n$Y!U9 zVSKd%iUBP*CF~AerfG!=j;{M{);-c0CnM)R{kE=7tZCk6BFcw(%qF;=Y1bZ$)m+d= zT$zM7?1(+ptajE|Rni2x9r+q2kWWD+AnsVbio^8f=v_#ZZAkLnq@UGnl8}2!gP%k4 zkHhwp#&_7#w7MZCxq9tAL_+Bbeh_P~#2zrQ20NPpacCb^qw%XAZarq&9;tOl=TK!< zLc9N0bfpC{=dgArwo;o>a|DDsh$tdf)iX>0eD%)xWp;P~!?P#)hidgFE;=;)17HNR zZaXBa$gjzv!T-U*AE?iklTH9$18Psf+07B#bCUti$6(S{c0^$lMb|0HwQwM~z94{u z46KN0-{;uFBWfgM)Hc)6I*q8c1#QRB23xDlsQv;{S04};jy(svMyr`S#;w5dJRlxE zHU!{rA@_x2r@=31#9+jfa6A#v)ydsPorz(bK3x1xD1i1`PW*xlJqZE=LCE!jr+-ZAINp(+%MDyD z=DHYPR$f6;9*oxw`PellhFgq0_5pcWCJdnil^gYYO)V$|&!Y;oQtTo?p=jnOs3N7F z8i`T^VP*lkiP4bhQlrFB$Rwlv@Qq!0Xji~v+K)l2KbHCOBq3#iwX-1@p0nEj)QCEdv#5Tz8IRlBf^I0RbTITaZT^h~;dY1v zeT5gJ5Z2HxQjT>!VyVkOdSF&(=AeN_42}APa{h4vAI~qCL<5$jC&9zfAbBLMjMI?o zg&+^VUr||JX;yoo4DfK`$9B|Y)DQt#KxSd$?@F;9={wgW9CGP9l}xxXpnxwA;^7p& zsDb)Z#=P7}t!^AB7cIK$H1Xq#p;) z;Uq+f4V@Dn~7yun?c=Iu!9TmE6%9sMTO^KxP@rv^>0_w!%ETQ0uYC|Xg4t< zH7eJGGL*bFTpr|!jzv+hmF+UImQ|baPaEy)Dq=?8G-X1xIMVueOQQqHTiJakHZHr> zj)CD8_#xbk7}{&w4oBBHEAmWcYE+bsVBg6nW_VsA;XqtDBO)xt6_x_h&5fIncy#3H z`X)P^{w_v_Iev5?vR_{-6}d01l~S}4Ck9B43^zyx@9cJ*`rFjqQfdPMaa#|MKS7~u zNWZI?MAN|a&f>M>*SFfP`-aU^(KAiq2^d$MM5Nw26|`-<^MH#7?94qD4>v=eVnWH+ zk5EM3ciPV|Eb(P*-RL_2@o#v8@&iG1&79>?VAUL+wuDQW*&tNzBCac0)_d4W&66HG zo{;g38wVc;b_xD8fwh8Nr=<#4OxZo9xW7n{)}x!6AhkGJdDBB}BDgpZbGT@C8KBiE zXe!ipy2$Is4Y}Phn+zzk4*RZ8oCkw4+FA=ltM>>og+uYNNr1R&J$?ih4-yMT>ME}r zdp3{PdZH`Olon2d6lsBAfX0sxnB7Rs{>~&aINriM6YFtjGZA>GFZT|y{-im zD_f?}G~fy|GcN?=fFzgCB~3b>m_)f>qln^_kn(!yQg&%;w!k9*y}za^jkjXNQNbxA zKtX@(Duv~{^wnDvt8c1`(?kd2b9BHfK)*Z8({;YAfxFKjQ1z0w(k$dF>1NUg3P$2# zQyU9bd&3xk;o|mMacIQVa+s4a4Rka<1D*q6WxWzUiruinSp5*chQC6Y@?eZiskCfA zE`!w^9^1X330-F022}gG%sHnqdQGVRK?A7q7etFm#Az`?!amyvPR`K@wy4cb`BoSL z_4s0YX+w}W4b$2ZOa`QB1SeGH1H9Y_0~D2MYyi*|S4x;aHvK^yeYy7xmKw@yz0RiV z6n?K58v6;+8pkAB7Mz{KPJfdz;0z#`v^2VbsMQ7?Y1LoQt7M^SL6c;VW)un;4kcNr zPH)!WsNB5YM7I`tG*{jO1zbS4%Rd^JzWTtu+)?=~_8=y&aDFr*`@?T6U$tjST`j!m zJS#k2#OToQr*=i64OH@WpWPePnJH1xa4Hbc(S3{!bo*^<6gOH;v@3bWnjA|kz7DvF z+H704qm+r2ujZ@dq!V)}F(2i)(Y9Szl8U*L6~M+0YjGIlup^8Z+E!6#L{0BQRxuG( zbmhLV5)7pN#3Z`l@hk*KnFi=~#GfhI=ABgY#!b;EkBR3%gu7z|G6qD~39o%)Ky)D9 z96>)XM!9f(nyjLEScfUpqk`8!awQY1mfDQE2=yfds^7b4FI(ME>@CjZjH=@zM#Y(P z=~=WYm#}S!9HXvws}q4KP^y*{acbU)`3?}b6ZJ?nHS*Z{J|NATrViZiBy}#-NH28> z(Bf-~-xg}tW#v9}T-9fc=Eazqz{PaS5z6ZpsGtoT0runSJn5pNf+gB_hh2Qxk*`mn zvc3fg@Gyrb;xE@-3vM#WVXt$08{Uj`Ho9a4c(;rs()n9AXkn;+K8BAqp4^`-*<ev7Ck>ofegq~M)$*cPKV$^eOr`n ze?M}TP|tA>VD-mJM03jTIK~43gDHF&&Y;kwhIVN51jKdtR?h0B^D$|mzAh7K4Q02WwwZcMEd8X!)37sO3DV(e=f!sNFZ1;66@93tphBX&v0JgO2bfs0rY2{9S4k zGv%RY_d#^7m|sEUQ=mhCklB8W&cb5bn#oz`nhv@coI{c`slhU*;A?=m$&A1C#|?HP z{)RK-eHcld1pR-y$G6^j9ifU%#t)NbUJJ3d9MDj^4g3Y;CxIQYjOWp^8%cj?Hk6jv z2i-#^kriP*bN6UH3Z}CFBpZQ2bVkqt#AA0-WDsNFYfX`tNUb3%+q#4r1yHFRf|}Gc zNJRVGgUzUo&^)kp<{G<&F&VWx?*R zJ&(QDt5}4DtI=ww(!Z57No(-Wus6bOhUg-RHXmV3759ve>6SU&`!>ZNlbY6OeL4FetbTaL z3I2_xtiEKD!*xo1Hy6lP(>KNk%SBIAXK;|Cb(|i3a@)%+HFoga zRACa`zCegQ6Cl?V`iMtl19EL430i+&t{YiR_`#%AbVwQifNf8Z%!U zs|tr=;T7@oCO1+Vg1}o^ZBYD1BMgtTvVP&{>pg*|qg9 zWZod4G_;7zSG4hV@r~ZtLSi*kqg0qfRHsl2t?>?yG(I4uE(emRB?n504N#X|%cknb_`61TV`4<@CHU9jnqJ#}Ix|aQ`>F6AoNC1!ir)7cZ65j> ztpit3V1>^jF)NM}`evbtgg#arPv~1CqRIG@LWD^&{MVf zU`n23Run2IuZP#J(1%l94Ogv*OPx{W&qx~j1pQjw^HDNTj>dCW6>!gBtd#-oJ z@beGljU)}u4Z9+T096NW41M-YMjZguYxl-Dj)QKra#l_TBz7)98Su`)W7aIYc=dYZ z3;C1Sc{hX4Zh!9naTna53QVROXNkvs^=kwpGcL~Z1q2D(+t30%kLCm?(i*gcJ;1!j za{x>#tB>y+OlYVkDs*Hl55*-5wXDFMG0E%@O07G5;wAGy8hsikK%m7RHg z-<#H+^1?&YQh~{It1e)S7VE$yDOu-`0D*xxdwT5diso*$r--ULFv;e8QB_=4oKAne zcEn}S0dyCDK;oh)+Q^_(wAmY^Mb!%YT=_0xc)bK&UBEmO3uZIPwzt_;-XNJgCDW?k z36OaD>C!?5oS05`VtmuG2$mfk58LfX#3cuDyG({+W6j@Em4?KJxtRCJSxaKhlZk|2k{kyin%~z8lCM$qMI0Z{$c~Dd#hKQbBI*A^v z#5?hV);Nf7^+y)jo(AO6nKniMX9$|;^;#5m+cDW$$6nAdg_qH)I}!_EJcbR5y7kpw zq%#>)^iCEX*9z}#1rMqMp208}a98iz{({xikx^w)S23I9ta@Y=V?gX(XG_NEnv79} z2{L+=Q0Iw6aR|#o`KB}JsS5RMz$iaqU`Sn|OOf-3VpYf8y zV#-5zOTmlM#YK*`4T-q&ICAK0^R|6M?&HHeSb*DEslq4a!_xas580 zH9yC))?3tM%!CV{1hvW|TDugV(~D{W<_m}x6RUaIj5#=UAYLqoB)02&1lBK0Plu^Z zqv>{LHy$_X!Ik^BKyHf!Wjt=7B@YN};2Pu%+LF`WVb2flfa@!G8<6#1MM$$)`qQVL46&@3OcKGKXw#0%xg{O9 z=`@m@Nwk?lXuc26I;8miCYwTkrE3+MbHP-+_dnEP0a<{4#1*91W7hMp3(^k6T69gu zgk-s%wJ@Ezd8<0Rc^wsjEj;zQG1FwrjntGnQxMmaQY06C0e6r(9%V0S`MGL zSNc$>^I4UPt7hTh;0v7u`|LPv>7R~Vkg^$$+${8}vYgm7=o&#Mw^+zap5slNi}5Mw za=Y|m=5{y#=P z|LW;>Gd1;naNZY$QI(o@ZvDY&{zT}+$8?_^ZX%%-nJ9aN72SaGMOZbZYu5O>S}bM}6QyhpPwQ6Cweu2!$$C4vNHq+mo$DZe?P3={92tMGLMr zGp+(rz!NJk4fyd@KiHwUEoVsoZeMxPw$u(x4NWSVb2Cl21?rMo!3~3M(yCNgnF=X z*mI;ewcGeINz7+Bx-Nx65s?-*x;wCk#|4t&^fpEni7y>GaE4BTna7|1>_!>v>P?1< zW(#L|Cs+WHZV)BdyuSP(zh+{AZ6=A*Op8(>U+BoTN00W^#VOQLVwnWb4*vIn_vbdo z=5b)%n&BJn)HO08>cI5Ay?J}> zQ%mcM9d&f8tE9bZ%`gU&Y;Qe}f9WhgoeA3-#-U>piAxYy1n9=lqF%-KkOvD0!z5ZN z8_k;82<<@ZmOli8;#w0Z$1CrsoBM-0II1Pm4JQgcEy&;N8l@{bK7GI0-|Kp-w7=IC zITb(3jc&s^f#pe!BuV}Kg(j&!#z>A|^A&%-{<7kSsla5q)g!WAIogA8+{M~3cv?Roat2|;sAVqL!2m<5Ur}1QWCVF!(%umnb`JrSN_C55o3B~8Ir(f{ImA8MR(3+ywppOGi=d{|077GMt zSQ0WYU8{J_Nb}{)u&n+ts}ul3o;N-?{~-^V)Akt$5^UnOd=&>vy{zrEBZ^K14~QAj3h%zXtVhN7zQ`zWDiNdu0(yxXD4@z`cp z^J3KGedo_Vbe?% zL1rH8{dC>vEjq@=8yzf6!NT!uH3}X~5{0;3HPfcmiu$cPjYHq<9=C5QFqv-k5J&+o za+hOX|MgD?Ok8WPRA4fo7gd?c-bnnx^By8eB0I%S5T1-UJoknfZNEeCx0@brd6P}% z>gH$(U9Igm+R{Y7C&j%iP_6@?#9>Q|!%6VGj15%{B;0J0BcmP6kJs5RdwO#}c)t?^DL>+EP zW~_x0mzzJyk4%_Ef|K$gF-oL#*J=Fr8@gMl24n;Cuka>Z-PPrsf&bm=G96jQs zuaL++l2bQ+?Y{q9vS%u|vUK^`&kxuaMDg5eJeXwrv+Xot`@BKYE4sL;Ak!a4&?jF$ z=Iu<=I>2E{HA{Lh^>Cmt?+p&&Y&4^j;PG>}yJD|QkP4P zowxkGyd9s09cYA|lic)rzg}{qWiiaUp2$yY~6v#8)Qlu6CDNHpd9- zSUACn`O|(q_w2c;;Mza_@`VqdZcAZdm;<127gWZlwlg>;ly?`v1&%d68&i@3&Sab9 zg84YR-0zSh5exz05fU@Q$w7(sxIP?BZhPYLzr3`G)--MfIw2Lg@&MfO<#D)^GpB#= z#Fuy97Z%&K&i~cP<0fv9QcuW$xv6XyA7hSBqU$uiB)W&|cSfoNoLhOtFtDYu5Nt_I zzr^LlY-c9yz`L#rDlWGWeC$}=C0(e~Sh5Sg0^xYBW;d>6D@<&3b(`r{(=?yxpL>Am zsdayGs@b~hmy8J93r(hm`eFir+DM@4uRvl?-OcFsi!pT*;7IKw#^j^J>7NTn+ zSUVfEJJxy-1`i2o?j;P9$Xb97$eb9RGRV2@v~V&WPBWP#a?bu&rOVX9F6OH@`}~n_ z{i|t4Dli$U!7G4M_zZU?Y2+|b+zGEdga9`ulQEs)IjBCA4wfvwX4s=2OikGgF})B! zhnHdUsc#~O&8IuW61w2@-5ex#n``lPU~HW)63waH!yxW5>t_;t?Y+JK`K=VDm7Hen zX?KJA8hUv`nr}3wC9nw|unyB0BN|R)Rt4+O)9Bn>EssD2^3LE zo^qGVOxgQb!Q(ixTRk21#PCq^P&0E3W6JK6jAcB_P#p+rv72=}Wgg3KK?M zDf7~SqqzRq-WXK5NyBqM5V-n{Ay>lUVES)LC(D3?YrqoTG++WJ zx=Jk$#^_|uI-DT3XC3A&8tsnd++IQ692C(S4LhzwC-f)ck;j^Sla1*_1X5{M_fQnY z6;s==SSZ$~DpLTL7Z$e`XJWuJ=qg0B6Z$dzkmRq3S%0>nQJu^N{;0X10!jO+KLE1?bv$@mzdMzQ*Kd}BXhB>{?<`#XmFoIEc-H?WyUHkW zxMm4oWEfNt{9VG)r4gFqETF2BV6(<)Z+?C!93_K2E^$LxWftADWc1~;7}coaF*UkBdSug^iBC455+ zUIrV!ht>;BjS-w(ogR#>Q(K(0Yi7JoQ&k|e!3`3>P7lV`c`N%BO%mr^CyVaK##iNI zI2*qqeXj#!>!jiaB>=&^c=JkSJM*~USWFP2MgzQInINtQV;dMtnXCt+K|6rxBzk`s zmnCFBvK0ssAw3IK_W)z-P_8rWJV!MQ(tvc=2Vcn_VWfr-Hv<@J9_ql@I`uYUYjho>`b~~1lVHMI=XPE=Yccjm_dd7ZsYjoz1#}e`42W^T z#^3t+dsDUo{In&X-ahN_lT+#jKqrKzQBdp(X;Jt;b_*C2g~dQcC&BYw`Mp}sT7(D7 zo2E>M5JOKf|&xNzk z-D%EZeTHNoPE(7>Sq(AwyR&ZppRc8l-yF-rU;eYx6Ia3V3|_en9aDFLVakId|F8tF zn+;{OHQqR-IGX{o03DPAB@+oMkZ@en92y^@TGb(FIQYx|diVV=UIydJ;cK^ynf}^L zP~ufmD_*AuL+a|T{e*S$`3H2()%l}f@T1#wZ(^>D&8k(W2V?8J0nMw~UOc7QPvdqu1hy>xUdP&tax#^VOVsaj9cl zHG;U?Y)_rk@nAUE)_q!CD|Nap!}^=e{p76iI9q)7joWTp`|cglTMthU#@2ZrNAi3b zNyn7dylptI+xK_%U|1&?6a9uOHin{a%C#VB2W}%II&18*Kea+B3Vw!akv5?y7vh>n zmA|W%$|e()4M(!=$yzeS_GNa{WikeQ`kI03*T;HayVm8t7~Ii}6IsK@FNH}L5w*)O zh1C`Y{+=rz5S8r#%3B$myun zc*bCCosRwuv8Ue+8KceHYp%{6IS-gvQd<~Xr&E#|!eQVZodlh)-8=Okckhu3PT2Rn z^JbiO4Cdd7ve39Ovi3CN8U!X>f)NjCtw!7uZF6!i89G)y1dZFJ8jFEFS3w2$5#>o} zDta~vfs=T>4!2WK)U|E-!)!d3M@6YMM{UD%VhOL|;99MIS{N17AhMZoYOqW)H=%`J zV8$^XscM3NIDu=XUp#nF1E=5>obnSQZy%cY^rIdHzAy!J65`gnMVKp@CRDtWhI{^Y z$J56)!k+v=*Tm2NJbbg1dI^mrYO%ij8_qmA_x-IQ`j>)|2@ul_>xT$(&(Y&Wnhn8l zndc>AXL}9W%01K^N{&aomg;(q$s;llPfV#>1jMa6_B>|S>#yACp2ugW)FkPE12HC= z=t|x8dezZt{MBR$t(P}czCnqyC59m=o$|9CTd&3@{O>1zM#dzxvt9VOE!v7XJkKuF?~-anA0Nc{4jTqaF*0}5zDK?lf)yIFCZ~S zMnV5$0px4UCnII8fVr7uXkfTDu9xRx+;tMn?0$Ypc@nH5kKAzJO^^MSOZ-Jb0mt}a zg&Q0@NQnkeOIIe*<8$zz5`s?QhD(!QE7A6e2*Evhu~DWz0!o-8!00iA82ek^Hj1~3 z9vi5}@n5bM+w*(LxRIH{^?UCtpSbhB+wfGf$rwE$B&F1Jkui1PZT$?iD#145{uu*W1>@gzUH7-5oz_!lTMERwhfvo!nyCFk>U1ln^d3$OpLHH`>0jR&p}X}@fQC!Zi}H|S zrX2w+aibos)G1|K(dSBo82g?#CF6RNZAmCU0N9@?*gL(+gR%9_CRw&cf;w~T2^0HZ zdtgXu>#PG~>I^r;=;1+ehwSe-oJ=B{X~M(%hr&UQ)OHy$8TE4kadc|X%Vg}zu^9}K zNo^}!?z(TZmgt1&-`cSA7%CNiipOQ}oTg-bE6jtL-D-r?SqFyQ&tqbR+^wpaNI+7? z=GRGZW&hot`FRF*uu11W9=VPu0T6kOkOt6evEjYXS zWk?4YCuoX;e+kZYU|6S)8Kh@ww1&DFDItlPV)i8ie;TGvd7wPD0#es0Vw-SLj2dF_ zH%$D^pe3J8G1s>d-(0;ev)-XC+5y;3J@WNG-QICXN*yU6PB6#_`fQZ!-E#BIhIXbOz{glnvh*V%U0yXfqY$wVZ zE=_uP%P%WpXV}dma2#eL!*SkXgp$RKgP?XJe4L_2blZoeYG5YJh08ReyiBQqnO!=J z0Z)0Di8RKhMPR2;9IS5b#r=mhd^tQ_ptoxYZ-?jF!vKz2;ap4^#5cxXir?`^$f~Xm zjA>0PfZ>jA^*pC#%?+x7A$1SqtSvKn0SW4)^b7Ao*DIypZg2QnfOmJu^bxo-{Z}f|1ft>N|9!ujjKh-jK}bHJ!7xa{yl0J zy@Y6b-wRNf@7fTMP2-&@cH3mod-~LIN{^=L`y?? zs%<+AsMHy=IeIX*PA$86RVHE*9SSvg?&FsyZ~Qo%-LJU&p!e?i;n!T9X8=^zQV+(~ z`8DEsq{g!{m-)S@BiL15Bpf3RM0oP&9$;7}S``0+)qY?DuBUiNdU^q2b$RRLR-(rw zinjm>VZzyTtL=%>fO;@!ZuN;@{roR1YeK;1TpOaL!H)!};{DcQ6a3E#izy8%=K!&C zV;ZrO7sIEiq|gyfHzwBA!e+X$G+3IbEoF^#<|rcvY~T#v7pI|G3niIZXE>K?P!Tzx zP|fpLoJ>L710ak)FJdsP^F__G0u_*Jxi zXP0JM(Mp|CwiT9A2gq9D;PIvY1hy4b63pwsm^#B5q{h}Ha2#`6r?KjF*QwLM*gB78 zosp95aIk~6bl4^T-iFk`Aio9B@LO0T%Y$%lgNsCh7G4>)aUighcS#~;$f+2D3Jg#t z)9pu4H0VY!cjyg=u9J^|l?UMkJzgVB(8DW3+?xQd@5YwL$l`}JkQd1CmW6)7!gbv% zZo4rHBPA}0v^jN@zM$1L=uPDokSP%~IL_|YXlyc;P<4oDu|9|+a4?btx^~%k?V~?T z(d@4NMwmErdoZk1M^K~2TyD@IvQX>GY=KGe=)Lz`G;k#X!I%t5uY-j>2evW(UW|k+ z;gBW75Td=93>vtGwJm0%Ic9)bnr)c#X3LtqxyBo`~ro^ZQW>_`foI11KQyzpoS^q8Dd zM+=B+)@)mC#1h-=nH;#YNVPZ>jFKMw?Ss$4n|1~jBG%CF5tRqxLNhkJz`|kb6){c7 z6Gq1nrFat`pSbNlUpo!4YsL697wfavv#sXuu%b@Ra5pKSvzAmoy6@IACNA``Fa&n0Cc^Iegz9F6oF+iJ8E<)zMsH1ezo-SU|8od;H(N?I*PpW#5!Ce zzu5y6lR-ossk3@Pg(1PfL{+L*K$@3@wMC3&BK>5gQD#? z1}ZrIK2VUT17kWCevP$4%(i85!W^#_P3n)mV_L(P&VymK)b1)5;3st`(TLY+EheA- z?{7*QQycVk8xO{!I+mjASX7KUtLYop{+3I)zRH~dEw2+rA%=cdD0Sf#`sm%HIu+Ok z^K7sT0wp>jdL53%jUk2-&5n?2t4;eDydjdhmPC3Rjnt_B4#NR==v(0=s%j_4Xrjio ztD1V|2*6i3bmg&;ni^@^qNm0OMc4-JC_6QdYM9MIA%d(xjbt(kNdiHupZ~qz?3W%s zP(8*Z+Jm@TG4_DMs9U($&35Af5Iq=M=Wj`jU5G}Ww-8`JC|4b@j=>O_Q;Vo{dPleuipjepr<;zfreP=QW@%sIz4Uv}BSFuO$=PO3Sq zek&y9Mpr9rN_C1Qv2uD#>^^!OgkraO#j*r7!DKY?flD|ZNW&3M*9fB+19DMoaaM6t z!WK;;wVAihYK&YD5i5dAtgBGQbE1jGK2a;lnm7Z2i`m}Pfd>Ms+xo`*!^oSk|8^qw z5beJ-q|)LeAvyJlkdw5ap8mKcCC)?YW|=T`;7LerJh&L3>YZxAWa~E)p<6Agp4c9Y zt&;``b&yQLYGDECeR03s_$Iw%byn9Y&Bb6?=V}fOZ_Jj*Y0r{uI_Su^;AM+wA*wC% z4YObuT;^*9+G9L3N!L2g%Tt@8s$_ zJs4Z3URqZ5Sf@dzTfIw|n(Li`v2{9H#T5hrLu@ZW2F?V_W0Fhl2ac#wKSclMC!f>s zasTP51o&cM(i#k6Rzr)!4>pYH}`k@R3orbjgeP9g;#s$d6NO8xv7EhxJ`OV6Dh!{7fKHDsf z^X1V=^^CE!d}2X2k(NFIk9qtcKgD2d3))?DTD5v1n+qG)d{r}-8arFnvT7do;BEci zWe*w|9|4ih5D+YZy48BbYJx}zKI$Hft@=+K2OR(?&dTEHipS$@*nRbpOb;hCJs4Z( z%dAs}ua4JgyJtmDM2?UHq)rubxcWVKTmL@@uj5_E>$k%roqZcaxDAIS?q%b_*g8L~ zNe7w9D~yX5qDvW^IV9?|YMecITmQNspHCBkhX^o;b*t~nd~$T~U~HX}Sf@t^fyW|Y zGTrL>+I7ZYSf{o)>&WdzVP%Jdpa;=rTwA-<^xCwI!LZJENT#`ve8x4zou3{ecANC=?i*lM)fdPO*sTcGHh`=I9;7l&F&t&t$!GYJldAm7#mr_hCx(y zOP$rr97t$bO`TF31Y=}QWo@%(63w#+UsJdnk=3mZsjkyZ!071yeQgfWGvaL4`St2L z?W~kcKXk>b(}N+N7|>`d<2DU4-jM|`8vJ>9b~oRdFaf%OH66DU;6R-N95utJbES86 z;E4(2`l#t-u!gp-&H_CroT1h(r{+cwnXQ^fo|az5L$ z;j^Yn@z38x+q-SdA@wHV8)A2_>?TqU2k5go2Ax2&`;)XxB8v$;W(AB?BO^2kNH4KuAgU{NtoVWTg9 zGqF+Of`d=F`=%ZDIXD@`IZuv#($W&l5HsrOd+Wcf;SoQW~v>lkp1=Sdv ziOvGQb9*N;&Y0?aVPwaa9=xf)@&qVRZiLYd52bLcoZ#mESG@P`?CCgHGnsC+olwP9 zZGN&atonG=Djv}~qEGzT#rVkt*Db#K`|r&K&n%B30L|lQ4_t@|te-Ynz&e=NM-6Qe zHnD;~zs%$nBtUxs^Gu=@95MG@_`p~3Js5;P_W)z-)Xrg%+kk^N5w(Lk_pIIc;VPP^ zy_>BbjIHxEw);4e1fFu)YUN=?%Dx~Z#H`0DI4+adyyG0A|y`b(D zGtGo#D1C5!5X?L9^+>cPelQ5VWHOfU^vW&|A~1Kwd=e`5p&@xigA0OqxdeEO+vvq9Xxnj|AWE=VF$gW8)6~2>@`-C=~fq4lfo`8 zCS&IG&45&wRD)Y_WHJ^DK2N63m9=XU23DRXGpk@Uq2U7TqabnxMmTey!VhI^EC7*9 zaid-`HTzI2S2S8BdTy&UF_3FCD;>s*ezIi4Rn?22Y}`)+Ihh{mnss9o=RNU|#RZ(z z3ns32@AJGlwu-35$I*ka6r>atipT3TUIW28&BqQJ)Ggtvb=S)noboUp@$Rc7ucE`d zV{)&cZAb@;($Spb4G9_GQ*Y#a4dgu-Tc-;G7q8QJR7CFnM}r{8fj1I(dN8)m0qkHy zTehW(whPxYGqZyS#_W5+@h^1ig=hig&O@BOkzct}DFo}4&B2n@bB@3H(7jV?BuEV; z+!>~EJ&0TxN-0c(wNYb8GQqVhmpzGIrPOAe;YaH2mkM@y>5z-A%FRpp;pPPd%DwR* zwVgYG^J`n;5$~Erl|)Q{6~n{%E7}flYwU$2#sCDCVoY>L3$K7}fGd%@6iK3icaHth z*sESeXo&GQPrUF4KLtCf?-C|nrw3!}BogZS>N?FsRl#$A`_q5aw38ju96@zWb^RW^ zt=|cqI8o&O7krRDy6f$qZMVt7S~z+L#K z-S_mJi{J10zW+SW9?x>?yj@*gU0vN>U7h4vV^QzA@q>T-^TO$bxhTnJK%U9u_xjbdKYh%l|4FSIES?L)&h^@Xm&V9d9u|JFeR%u4)xG`it2 z^dN!YD5ZVq)Fc{k$i(C7kK2<(j^I(Mdmb8K+OuSK{j`Sri<@Bcs7u-Q#(%Y;&kqj{ zCd|3qxAV{_<^b?{LK0Kwn)`MB;O(uiJ;ACoj-T_Te_i5C4RKCL&V5M1x~7UwV^d{Z z!2ys8=o6sDUZq?7;ltQ;ZdG0~ri=&)(-|rm$|HFpLp7C%)-6;>pM2iDS5N!AF{Rj) z7p?nB)cs4qhl@$;isE#bkY{yLp1gA4(P*_s5viHS@^mb#nRM&jCEV_LXp~Q!AvA-b zdp@6q5|7LZ(3bd|#+Xk00gdu$T>``ds>eZNZ9cRgnOP6wMgnJ`O|O&ZW>-TO2l%!`;$H=J>mX{}n8EHdUUu6rID#oVbW8J`A7 z72wNL$0>QqnE70HCr|k)c9@>9D@L|&P6TEn=>mRWEPq4GA@Y&~R~HfF#oWNA#ucrL zRq~?MxuF?}=q5n%-)~R-@-y=i0LRxus{P%!w-mG zw?f|sR$?RIFm-P7!I)iMzP&ZEF0Gq~Mls(2CTBG83=vy@a#ZBDxQzLnwa{S(G>Yj& z7HSN&?BXXs9x(kt@EKTMX>_FlCiRneJutj~W@@^kq-t6^-M?he)Z(dh{mGEsuwnmi ze1eDOWY!lDvO$!V zo`jL-D=_h00ev+YpT-L5Ode;EppZd%k^J!9s3borjbhr9?KZ!-Z)z9Z`&;{ZYbkQL zq(fzC*NvCj{%Luz30fN4{N%LhkTvuLA88@@$lS$agk${NF-yumOzlgd>)jTQ-|p7S zNJGg^We=wT4KSxd4#RQ^LvndU@{%#|S+n?*pOi*1p9JPp;GxjFFXfrx{nF149g~Xo znEu|SPkeE-^*6x7y4&edTQI%MUde9vT=9s453~RSK|D@vNoUB~(7Sy^UlhzXzuicy zRdkn5m({OxqA^=ZoO`0<>omQYN7toO08pNfv=Eyb(gr&FP0H)Gblm-;u4kTlQWH8< zwk(!p8`L_uvnQYz{Pa7-l$o@Zm6B`CPAj`EnLf5G6}7+Z_#xZXM_RoSNYeLq0vf?I z)4)S1=nK7D_b%Vzrt9B7kn-y7`t!N3RW-Z7s-YE(O$JNXQXqa>ZVyG?7QS~c_)g2Q z2kxHcavblHzD(nmGUmRd)&w+)IUVI9vWIwgm>f|?=`1*<13a}r9s!MFx;F7Kb(T=P zXztAZLk}m+|3aC383#0qX;W?7&%MyeidPfJ1MNQg`Idc;CmDxz*EJEI^$w#$lq?>n z^OANix@*(+%^NH_-!8@2rn-kn%?d8o(eu&`iKqCxOPAK0fb^bmRr1hiRc-=iU-A;&l~X@7pF7HD9@s@+BhZ;t8mvfAN!K+>J_83vU|xSWms2 zz-g^6Jyk9{8I9ZM)we^hWp?6YL{S3sgiSc~z?>AOB8^OD33zNvuk2 zsv4xsiD1(&cIC2bN@gBNYgD|BbAT0<^?i^K&YWbc<#}JS_wA((y2Vx{9}TLa{*<}H zb{o%$(Op7GXp=GfusV)Glj5OCt78k<&hdFG!I|3_^WUJyL)U@hH4m&+H?~dc@yVVR zZBP~331~luxgT)G(j&`7WAC35K}m{&jBFV@J_8!mJq@^_;Byb=^GC=Qk7(IJi$D1O z5K24L@yb(+x8Hw+RTI#sZ>JWg?Vys`>2jRx8UPGD^j_jPDRz%kTtfODd{Iz6e-5&q zIC^G@rubrZ%?yH}4r32$KS~_M(-~T|mpu`1O>0T@)=i~HZ6U>)RL&vwOA*WV5g(OL zT%?CD(m3l^+n8>-l%e)rR2Gj|&fbAS^Ofbqd`;f&cB2$N#xQ`<`n?P^x{KPnlL@cH+w@*R6$gW69;A7qO%-MT54%20z^{T3l5* zZB`lOg+TO&`eupu(rlkv0HFrntB@(ZmbMh7H7qTwE}1riig4|wtAWzrGv+yT zZNF^;8ihsIHV?BYaQ#%%H%hx{CLp)AQ7B7gEd7S5uvjLctS0^q?Y8!?59X0%vAkkM ziRpB=SP~GqV!1QY`he`6g~jUOHP)>Hs~0c`eC-g@&Sx2r>(ATv{wuCKFcl4*eEv^E zmmETx`~*wT*^ATvIpy9Pms0ZMSLQS*N}W&VO#^$O*A8WtPPyW-cXp+18%A;6oIM$u z%xDV5XObj(+EG$>X;ax80D<$k=<{y|ZN8ZtQh9+T&`w}hGQQJDAyazA9}bPQ*Nzq% zo2dtnXvh&uyWa2`jmj&w{l`mIX?kVrSzIK}t$~~58`R;IReozl_1MojC8CKqNFeOHk zm+UIM%{;j(t4}H~=H;Xm+pV9~tI1OxQjxqUAFf~fc7W4~(i$*NN%~u>XZ$@Rs;Qm7f&{UoBrIg@OyYv+3SVL4inoae{Q&x68It0yCQF#K^++05h z9Ds}oq!rL)m$FSzN=@~&xFDM#507|%ha&WNqla!LB+Y7%4t6a)Y8&!5R694HRWmLx zB*>b-A;|w>ZYtjRLNXfK5;tW7q>=r!t~TWF9qx@T{7=rzYXo$*V20>*np8QoUUkhJ zxfodr=`Sx?fugR3HDp6xU}mk@uxwS^T0SqXuB2POry*n_&&S};j)QN*=FTL+XASAP zZc){@Q@vXrIE@CRz+DSyfNA~YOVEyuYv>}zIKzQ6E0W(*Z<$0ba8+u0B)l!`JQtVL zRL|h#&*b4!HS&BdMVh_@Vam|E3(YjVje}3MG@IY#Nz}N|&UARX z^VDP4j5+g^3A8Xi`}}Lxw=K64`V*Dini^ibUK{y_N5otHsZ`W&!_JLA|Hsj(FlC+Q zAK~o(JZIVPhgNRGqIVDoE zf^CN6&XjqfIH|4d1-44*Rg5Ykn>p-IZe-{4vJq)0Ss!s{ZtUXap;62`sl_%m86K6< zlJ^_~+kO+Ii-%h3k9Om$pxdt%41(2g4nzuuX#hlnYji&PxD5}3J z8J4AE+D>6UO*XYoe>mv6f67#`aGd6|Oz57C7vJnPZ*=F~)w#Dzb>`V53q19qARqC{ zGof1^&}g;nh)>5xj<2bt8y@}68*EKwLgfx<6n8CE%iPN<=$rta!O@W$Sv!EZoT9wV z&5?&*@z(+WFZz^twmVLE#D{6RrQ(IuiKGRmU^3)qaFJxo(XXE_{AFH`gV5XT2+^D% zYi(~MQBZG7XK9Aa#-i(nc38ISeY?75L9By06~*`m1!Nr}y8d@F{ozpvX)NdJxfu9> zMyt0MFy8@vp-q>_ITQJMqTN#wCkGl$m<<`zv>Y*+FegGjeovEJf${09 zjyX;=Xiw;S^WMF`eCA+E<4x#mHjoxdS@qMfog+T;&;U~vf0LVrAXbuwL!O1iYcBnH=_Si? zbF-k~GvUUF2V(5_1P>$*WjGz?WV$;^1r{jr_!c>Es>CwLy6`&?LO@d&2UTU&m@?6M=`C{5j&$1K5vk)RFFDpE zWv4DaW7Y@*&FQ2?>8_YiH7~8CWL6cWl)!2h_7Gg{WW`07Uc#tr9>ZwUODoCO`N1s}G!-icW2H?87TQIh9JCK~0pUb^#4a z-5S!er$ggVB)9d!(xn|8?rGTddzGzj9va1LKrN6xop6IC17@YI>%ho)XcXC=pO06O z9Vc@3IX+LYlbs(XYUgk41oVQR#!>)XXJt3Wip$A(lo<_T_LD{PD6(9J6H1z8;##-_ z!U2tzG7hD*mr^QekD2%IWjV?WO_vK-Ro0kk zlqx#T9zB3_M^pBX0deAM%G6dxYos={Dz2BPD2r?|Atwa!8`Ocwm_OL!<)J~PPW%7A zQt^#ZxH`+$>OeG8tP05{pc`tqOa5+?02$J?^wt=|8o3OadA@-v)Teohv8kbgGE{1! zcNw!4QIlCeBA*s>JTL>Rq?V3{$RDkIt*l(VeEfi3d7cCOIB%VkgYwMAntgO?>3Cu0DE4pj z&@28D;6G2F=8y?(L5M$`L(a^DN28c;5oY!nkmSU-o@9g^=daMGIVA7VYblR_^VOjD zTa%3oo%?rZe}cIEfCiYA(0+tEmseU3RV(2O)D>S-7dGF8yC4D=*|RrF?V1NT;QFg*S&IiF?TYFvr^I(WA|>Q zo#!NK;(Imf?jLme&>JV5hQo2&CMWy>jY{(sNXMp&_E+?p|MQZ=zTD=xRP^kSWixL3 zV>V$1L072@GCys*!(OSVf5p=Jj}+1cXTiZJUXr9nd3T@J8gx73(3DBD?qC@>zEyqI z!Kwc&q&VU|U+=Wd{39l%s2S)u00n;$W0cg86T#Sg$0SI%`m1r(5?8D>`WQ8uLlCqL z@1p5=&5y#!HnwQjV&ZzFeTJ!_7_mGwq7&)0V5Siz-lee@F&M7R#lKcZJ7dmf=_fr6 z?>vdwlf{sy%W2WGA*v0a_oL`Ved0M@fpw{@@p6NSZ@kzL5DOB%U<%mIS3~9T(EOzu zfT=8;0Tu#RMeUosz*Vv1(^*8H9oFcj$`EQDsI?e;W}1R6KN5($11&JH4pV+Eb2_yK zDMPZ$cr=Q6DlpFgeW7>piymHLD}y@$<3__u11~p7;EB!zrFLJF5i#e~X}zWU*44(| z8i#UuEx5!LdcCaWUjRqS|tQxq1< zBpOlnv$avS?R!y)GS-fPF!9hKN?$Uj3G1I@O$4WQFXW7%?4Nqv<{SF|qzcKtMk5~U z*=onx4&V%f0SS3AeZ~`IAyazg%=PgcoB_tyK<0~TDlg^?7OSftm*kC-A@lOFqM$Xc z>BkL#W*>SG^X}j&C(NL5vun9L;oA;-?W&uOY z0}>B+^<9HrSB+@1@Nm1lZd-FQWENi+Hwy1~#EnSjh+& zSkERy&+u@$j(BVRPrG89@XSMl?P&{Xf~~k45B&2{;&EsOQ0Y8Q|HMtyKB%VR0&x5- zaY@SM`m7p2*?q}jc-0OR#_@Vmtm{17g!Lp_Z4;+lSE1OKx!9!DTkqTZI;@^YqgAn? z!E`ogDz28s1%!EfdTjJvtVh_>o8_Jkc@3j|bR0U=w&+|merf08KH8lUg z!8!A%*pp4KlCaY|QH^Sw+hozxD0w{JFqY>-l`W>7rU8%7u6?=XX-!-<)N(k+0gYnr z3TL|q0h7#{5O?+w8-&7?hJY((9va0Q4oo{hhkk@^f3Bp_G)F|!Z>t~T=z zDKS-8wswqb_m>S;a{zb^4GQ%c!4;PWfm`eWuKw<#si%$^oQhuEuha4MXH>%%w9*THkE60Ev>zie2Ke=<|OiKQ;%Qt`C)q1uwEjHyxD^^t}B^{F_0vS8twt~lb z70g4gO}7b6cOUUBO$YlGd#L0UIfpH12DO+%qnO(hyXHVvF=q~CaUyzhbB95PHNX6I8`KyuBBIeUslYmAs-TW-BI}~5%JJId?-{0Fj zNqJnP@9*W=7TZUxY&kBheoE;T-_iT{Ie6fK?AoB0K4ZUY)fg3NxL3u0H@N-s@U;bcI(i(OuY z^7|iRnJ6>#uTyixu!p9Iz*}P!a18tL>;XNP(G)*sDUC^4!;!}0u2}iWr3d$#N9wZy zo|f+w1T>0iQzrcadf$YwK}gc(>B_tI%WDNNo4Sg z&&D2NM{ZjcOfb!FT6K`A&Z{e=HeY!-1cz3s%&-biaAFVehHriHAy!k2Dwzf5q11u^3a37 zz2Eh7B~RxhS9#A~JAU$z+rD&y!U;C>0=K!V)WeM@U3TEwC>4ELziiSSt5K@2B$BsmHxrPhWD)PZ0j|DA!Nr8_)U^9vyH&^br+ z=^pE6J~()A%G5B4Q#Fk`oASfHUtFBpQClVbAHY1FtZky_;%hjhZ;O^Y_%oLiFw>)=xznRP{Bui>h!mf(}NPCWOb*q4*V zGCeI~O6SmGT~z`at;%D-{DZ~glwAD-$-9`NBLp;x*#{kAJ(@inF?3A}8|Q&|Siw9R z#asnGze{4WPe6g>MO&Tn+s4Wbgc(?&fJQOxsl5}Dn7m*^k+^4HaQDfB2ezON8(83g zMlowq_Z~^ZqAJf0BIVi*4NE|ym|y0UC$}l7lDuS0Pqd?NFXbntQOvLMV3O|fhjcKj zE=-R`F~81nBM~P#H4;IJ)goK zMSKSO8PF)E8(00l$?~U&*mkG1ula(;)f|`sjbgT@<}r)7jhxL<4{Jh>c4ML)MnPm;k zKM6H`94%+nHY=u-WM!tjtYj{h-WI~z1dta@mYY&vP9NKdWYMyi79opgr7KD)_|YIyTb@s| znR55(mQ*2u$Cf1GfVuP1VvxK@`~XsP(gKe)_7iOskx;#rARB zXsgSMaFcfXuD)1=EFPBDB!br+8#HU(qS88Fm20Q_u%Fo&rq{`arw@cs+D{`X7vnjJOL$HMb=%EEwC*qJyS1%tm5b>VUz(JI0`3UjIf=}IcZ?#tYOV)S_~DF_|B1G$ z-IJG0$hrI!FB`s>&!j6{<#c(|mI`+! zNrx_ZLAThJicfc=rlr&E7LVE>&nMZuxSlaLU3S8jIJp<203NyzsxL5M*U~7F9!$zX z$o?Gq$$uh`3QS^wGGw2xP4t+pO=?x_IC;5)Y<;x40`|BHRaW5tDybu6LI4Y zp$2%w+beXIo{{-1{Z+(A`cCUd!d>a#;U*vDvrig$02||c#o6;v0^W`gd zkV_g%cZR9a>{plJt449SYF5Y8rA&D-i@5K)!*DE1S}KwkgJa-?k{mld>q79~?V7SUIBaGHRxn%Kb!hL1k7YtVL`I_PYs$ricEZp!|DyD3X5 zB75?{mCoj$R+IS=;gt-2N@=nnS{}STYnWjQXfFmzrm|e5UFD_2!0@9l-Q3bWm zuF~Jt&J)W_Wfip3nGjufmX{oJBG!$+B)pv3Ma z+TBUn3DskYGm*ctpc)tnAXPNPIE74$!7hBN3~5prGSg~otQ<^TID{nn5UKt2^Y zKxva6BbCax?Wj{VESWQhlBB^G;0(7+w%?+|tFc@>GuA@jIW*OC(=*a;jdS|V9_-;dS z@rxfmX23yh;4xbIw-}$+P_pDLSw=MPh&J&91CEu0eI3Su%!X4EjQr90D3(E+DFODXoo}^FZ%+b>&kW+DxcO0Sz#%utr0| zW0HKT;_*wp?BT%lW;wWe1@wxK#xwgczH2+CC*MTeet4Ct*ro&H=b=}8G`887@tv7a zesqf|3%@Pn=b;DwKLFhqkCW*^GI#XS^SynaU`z>UfH{mxwu8JG&OAFAiVDT~p-ARt zRZ=$s8epp2Ho0AWT=H7MbI#e-_&@#mzSl|BSA-W5 zn_j{szY}potPxv@i(}&3MqDQfXizDO`8dhFwqbE`IkM&ZINF}F|3nkK%=mC z(P(MnlgW+^w$yZPn?G!Abv{uwf1VT2D}D)d{`_P=)wbggX~oO0rWwmR>G%)mfqxdD zXOSJKZI3x7wQN*|VLt-LKcL+{5O!-6j z7C(Zfz&l%`0|ZO{k0SFklekq&UBDMwb2d!F7UzXH7iQ{XNZ5*p8MoM=&CKFlPpW(Y zw^AM&h|prvxkcfq<&@Sdk4r`8KRj~9loeM}ZR6u>G+C%;^9n=(jbg6C zFtaL&$$3|Iq+|dmHR;ghSjq;wEI#kn2h2tpD%+wEK%?l*%**?*WZs_5|k4Bqlf8ew5~I^KRe0{1-6Cv6{Ma0~*DI zfi;al@5`7qfZUG>)6W<;$n2X-#sQ6DB6XBm%U-+VL9Qf^uMfkOZEt|4z^)AP@}O!m zr-&PpGJTlDSq|x#v8Pkt8fkkp@p2R!+$Exkm-t@Zu~cjWh|pJSH74rNewpNK@XW^! zD4I*>b zgkRe@U1iT>mQo@6K+Eyap-f-2`W5Icc<4B(9>>7jGMB<+z9NpGU5ZDBLul$X&7+nS15bPhJpms(OU61!Opt@dxB7n|$W z+Qs;~rVnToa||$JL+OIK^t6Di<7G@UmPj5N#jGSusv!nZ8?Fw0rwPCPuTqvVzDg1xxzd1w^VUa0j?Qkg7mx&7VZx#PRi^<}ds zzIOQU!-!AmSgu~angsNU4}ZuUl$0&sQBBEm`<*fKjCS=qkq;7FiyY7Z^K@v%8F-w@ zYU0euje~dn@$MZcefopt)vt^{emKeY2hjKiAfOSY(KIN3bPep$h{T9(Z%8z0AY{L5`J(MwJDdFZ-H2qo)|cJVO- z8pX7peqereC0kx{V9HNw_7!sh>evhkYU1ozq$7@BRyW$%Z1(NIi(fB)#`?friQ{x< zE1RmS@?!s6zaGK2TIRF#t=W^5CaWxsZ|A6<=DB1O&?x50oS5og%1aK}$WQTrzLm8Z z(82Uuz}$=ZTulGK$I#hulU`r_*J0c3`8s^o#zaVMn1@FB{4gh`)=OR#)3t@nqSYe{ zRY!IW;@buOwEIQ2DkI6Fgd;aY`hhxOy2pS z?-Q`!NVm0|{M385mkr_@6D|KXl;8RgzJCzVEB?uZZ>Ne#b2+Un;oIC0Y0d@K)FEs108tO<2gFXI!Op@fq>r7hF38-jm$&bUC zYi(tn!Qhttp-9*2GbX5suEoep*qAw;U#qyTWB8Z4j9##aZh0bG_U^yRhrf%CWP*AH zG+HmGq`uOz5o(!=M$By2=htD#Y8=}~r^o?~V!96IcStz_7tQ@({ln`P9YdHw69qJi z`7kkDxAsHsu>;a8pehkc=8Pn;&Q`e7My7F1N2(a+1 zkyq|(*K*!U>PbMOm`Hfa#?hCjPVu*W{Xb8v9D+_9WN{5>fY}}W_cIuvrs+Aw$aq;+ zg$2`OGxVpVBv%hvyW5z32s&`5xT!oEfwpmrrU&8Yz@(h3-0?BzzL3S5EZ3t^Oso)v zHivJ)q00d#Z0L7Qt5*kEJE_$(CCNS?&?|m1k%m00{pPtC7VR%e@*L18=1~-A8}iI? z08LXm!OWPcNj?J_V3HM$w>x*&h*#gq;!TcXa=rQ&PntAtG>^YoTO3Z(?A%b6m23Kb zJh<%=9JC6QC!opZVmIXid0?ld=g_K5)!$&k8Mj3Z_I~relF4*6Rs2F~h9TE?0q+0D zg`2+VK(i5f3B4ORf)L*bILbO|PFh9EXL)5wDbj$_h1SL8loJ#gyZ;WyPHaQh{Pc9= zPwu6gz~UELI7&92{G>E>iOtVooRHfjds$o^ZFh3{CEb4L_Y8R<@k_=WnPftKQkrZg zTv+T0CT;h@C^S`7Cr9bd^euku+B=Wve^e@x7aixF&*Ym*AyfKnPF$d>DJv=6vKX9A z{rbXPn;%X?QzS3Sh;t5oXHUqKKAVv}R88QL6~iY>PfsI0YRgPH1sV47_zV73K`VCh zqD;Z#3q3a>Q~GSCdWYdg9KgyK@NiHAVz*qJqIwKG=5M5JlsHxhA=`YKpNB6mXxgk` zv*wmF`D+R>la1qS#yW?LrK>7Rh@tElLph1zj%0+vcL7aSQKksG?XTYIJ)|5v(1&c- z=p_3AO%~>+t~uDRm`=@ZV^vXwE)W(T8ib=S8Hs|;Lz7+H&Otz47)3Utj55jBl$Ym{ zUvs+Myx!oCpB#_L@72k=%1=s@U3kAO7oM)-0|qF27AlTM9MzMRFEcLNRXo(BI6^q7 zNqU^}y&}=a$tVerIB3LM`KRH|hSpA@y3)CwvWhB7AB^KWG&b0qDe#E*UA=u>^AO9% zMffm(-sy^L*$myXY!K3igBRS_IOIILje*BWwWBa zR$ej^ZkTM3I)g|5mbw&Ar?HeG9sS*EYDoL%Qbv@c8P;%Ar4pGtcbIL~6E zCG94T+Y$@D&lXV_=p6JR+I+$Zuc*x4ujF=(1LRD&p&ihe4fIc>ZccCkNj8Ihx7>!r zAhMc4ZaJdpc$P^g%P9!${~*|w#WkR&I$cg#x=YH_6tG@tY#`c)RJGc6@xPUE=CxTT z(3vx;{{i%yeZ<{O_bi_%3QwhMK}5s;hKSe<)3A$M70T1I%3POoEy|amHlXw}N4`Fx z<#ePK3Jx#GPfC;BT)X`ZUaAn(Zy9k+IDKHxpH2kG3sz~Y!cU_i%kG6z=$(bY#!8+K z!P&87GR(zzTms`!XwHJ*@zBLtz4$ywZCQCJzKHW(bV`{8^eGkVJOi-_8crX!AES^( zMZ(FBSL|b5X>gXFjYXVd=FC9`p_HBjcJe4*(91(FHBZ|FhLX~XStz&4?6}kNdjD?Th4gVC zyX3Ae0Sz$iK|ky#n9J*|yD*cf>u7`5B^47lVAZyCCO5wHwt}R=8~K6Pl`(^vJ zXcQCYcuhAbkB`a9LvFF-GtldR2ADV*YY-4>7PHbiZ(QbT(e<64X*72;&5VQ|0f4my zn4(c<%<};X*n?ora6Fn3=ft!#2pXazt6n)qA{`TC&nlc;iUz6UZyF@UDN5rMvJ-c| z9D2>P97AQ9zXr@-3&=v#4aOCZSpCWIm0fm>*Pu@$1i7{id1tg?MRN^W7AnS>MH1B* z$Qh5=`ym@6i*qJEs+aWSRtw1{yUk`9va8oim?0Mtz%5T_QzXIcmPaDW)pH8V!EvUo zF7f~7d6eX@qDilgrCC5B5>zK5ar#UD-~OqCprJY( zH;5$KS8)AY4-?(gZcZJ{4m-|P`>miX$akT{ggFgr6fORN%C7B5S}acMklk=rgx55v zt=+7c!PkqmFR+85?Fk3NPISn^s6H`9C^Ytr0Ula*oF~<*dXD}n+GlQH={P0g>e2yY z+4u>oD}qn~n$x%gB&$RHj^X$?q^(Xh+N~A0o9i%Y#UwYJwp)_<<$}4gNZQQ}c7D4t zmo%ctOIQ(YS{W&187Wy2vN7VEsRCY!c4vl`KAP>WY;X(44MXEbc5f z|AsvL!@w$Z=lt~H9_XxiK%kqqy}{Qqzx!9Ir5~)PqBG)u3GvnD#UbT zVYrU2#*91c_vFOK>&lLC#T2R3l@3C!vR5pwE;0XMRd>azyfk82bkWT<{DmB5f$)`u zm`kSyqSdigpI;EtzDDwi+wWk42iJ+8`ZW?iS$^>12IdCULqVQc!W2_DK@?*!dY-Gl^4T)eh-I z1`8|1izCsdlx+zWWF=x2Ze>Bbc48!&IWaaJ&7y|-1+{+&Hlmd*Czq*j;cP3b8SOPn zPYAoI{E{^g+G(ymw{|l0z#j;uViAYT>}bP2{z>elH`6fASD}U7l!lBhy~b=jee1TV zx)wf36InFfY-booa^Q*Z-Yk(K`~x0(qf~VEbL~bhK4*J7)3%xp589l;s_Xn^796RW zY}SdU!*Ma^GWjyRGPi>RzpF8#+O{=C=1wNb;h4z`5zUNBrCXpRBuvP% zGXZDp?*`-A#<7ACh9dH78qxpX{*Ws*G$HZTJk<#T74jac~3 zT~#wJ-45z#g%3B`oXOJ6Ll1n*b59~QxV0gR=S25F4&8Wv+^MzlAa|yi4_H)=MAeih zHym|q#c!`k3&by0P5X8Z@dq?2s(*W;aw?#rmASKu3ZA_=heef#9{7I(+FFmlxOs7h zufZc8rRXRMJBNKqRISM*kf!FL0VaK{$s4jZ@=D61%Bud=bgOsqz*59&k=_b(4bZp z6Zc!20%(%IRVq*sUvooN*X))YXA^vGERu~dC(J?#+T|6>_BqA_Hggm=AZ~$hw(dKs zEoCUf&R#`jMH%tuT+&3@*jl=etwJ6eK~A55UNwT{=*DW9KgK$?9o0E@GV{X)l#!Pb95A0wNUW4JXv&MhIk_Azjpx3hSeq(v6O~ApJaeoP742A6*}J4; z@odr+&Fqn~bJ1(C_G0%{kLng2+8y?ARt196q7*ZC;6qOV+6YGhbjr*Gkr5|eX-YLhV}F4 zQ9o#p8HHwvk6zjS2xAq=5n#PsQTvx`3R#Yecj9+Qnd=BJOROv=it?AWVB*s1?0?Y| z9jSUa=iw!CwIMKB5bAc^9yNY`=ShJMD+(L8Ue=Sf zoSY9K`~=4Ri&W(zt!UpxFWvRXtzDgcNlr}2iR1E{q|7y3f4B9ky+O|GP!P+U|NTr# zaS)45rNlu*RRbuC94(4Y!!gR5MP@}hfy|h@ zlD3Z}LCdY88xqy#v_>>tLnzG=EVbldKocu&wN|dN9z;#9XU<$_P*W#U+e~Qd0vhQ6 z>2{=wrXFe%b}*etV9Po1cD47)i(=xM1)2cT5dsPVPnS}9-gNbR!l#K?oL@sVLtb)Z z9Fg@Cw~!KK9O07Jh#lbc%eE1BLA7v3f{NW5eH@Q?3Pn?L9&vuRc7x=}UZ>fP&21if zt>R$d_hq~2`h?m=VGN$N!`~V7dF+ylo=T1oG~|kMDGQ?@6$xFs8V@-87 zLz+yd73N=C;BC3e63u31FBEmxf!|!h_&ElBv65mxoMtnZRCgI> zVym5=0O@yTMAx^pQql}2aG4cRq)8T8+hdwLo|XY&APfdDYqaf>^8@z&x9o5nOs1VezDSs@NbNo^Btwm=s}0_yxJXP35QxY^`8l-JCz$f||vzr0ZUze}PqxPX+r zsp*x5j;9xw%u6I*7JZ2g?XRwZLvDW)5I8dVsDZ%w!IK+LB(38ngmKnFItTodxM0g^kDEuwiOhRu?B>FfZ};49Uq#VDnO z`+##;WDP3fR7kX7Me1NR>EBe!rY6IqvWl{Kb0{p@&y=nQ+;#K4j!EEp7tvDJW`yTH zNU}cHoT8&T0%>)YqdKdp-;vN9y{wVC>X;{!DiY8r=3~H&FYk5>Te1Td^{-f3|B*sE zDi|IX(5dz%FP<{?@qg@+qPDV9e3b5BrGvUtE9Vv44jIpDeY3)Kr14ADYf^GTr~Tbu zQ?R=8c_U)^UT#$<235zov#xCZ{7+aq4Wf1e8WqFa;JFdf=g?6%60h@Il#jhGKJJz) z?`==G|AF3y)uDbY=GdH{B+-7Xy`_#_DDv4v!<0K52{2WF^c!A&rUvM^VISleSHn2UrSDmO2j=Vc&=M5-?@<|b~;9K}lc+L#+i!(@RZM`N~I zKdV=hr#esvj~8|G&?sgHDD8u2S>F)p052h%y6EJZHlR_=58!$&$~E@7mGYT~ zMlo@Tg&D$_PF5mm4JXn?{5&*@iRL$VB{5m-WQ673>5Qng!whH?b0Qq-wswAl3pa{O zC^U_H z;LaUeo>^KS`x(JP6GPGbMtZC)G$mVgy^AB3VmVV$yfm^uSimFZjuQMjlXo@uk~}nu z`7v-}|H|>8qZKbj?@CpP9iTikz_bq5alpJew?>ouz_MRMD^pvsjdYmulhP>WaA2;Y zwxpit#zqcyq3d-%y?EaIy$&lSK7$J@MKPoA_CDHtu+uRqvxvo$Y-wUmCRE{-irjFt zBE0=Bm*2nr;(hJfo;u}WZMDBkl!7`EuC@V<*7ge2_FS&5V^d}w(K%6*L0k2ughe93 zJT!_4@5J23t>WlKx1$%|>@{z6=iLc2h~5uq6!TSJ79{N<_jg)gp4VvJ>ra)ziHcj= zl^f6~CU!f_Wz47RT&$5aDLmnw_bxhRI_2q=TY&S>D5l-RiF=N-OWGN^*=bSH#62eu zjbe_bX}$Cu=ML$Vl5Zm3gZPZE=gdQ+m{=+>H*nn@z4GbWD45#?G>VDoj#-w(45k_x z^BeTdpEd+EiunwtBdzQdik36n&YXr2|60fCv^#KHr~RF5 zon8erin$q>1L5I@0|5;#{NbCV12{sK!wdFk6thKB!W^8$6f5E-+zEDD0~*C#0L;gd zeM-GY7wa7G8PF(ZNpr%)8MUmstF9i|?4H%Hq=&dZ<>VRAC?+N@=8mL2Q48Mj%2SKC z-yi+-Y}PA>8PF(ZcVM2!?e26*_Cw=Mt^N)(4~=3@2Idn*+Am+cE~53QF%1Lw83|u5S)x~6?n%Z(#0ToQCsRpK{+Ss z4ot>VFt_5XtjUWOix(N!4H;ya_iQb>#V8l24CDF>#Bp`FHMap}r*F z3u^bhyVJTh&8o0;h$Viy5h7FjB07254R=6xR21rODLX!fCP-)nS)*QN8&DCRC$ zx!RbNvAC)cr(wZ4;3cfqPR0R^Vh$rrTHmlMqd3q(>AsNx(0$6{UhS*eZ(zY6i2@ph zUkH}(XH}~cldSr(ILyE`Pfjt5{t5XKz*__FlWXsg7iGuM`-6-+ddE zcR-_3a^$9NZE4mM_UXr!objxyw@xOef+pw+^^u!7f!waFQlMDJQhgJT`O zGij)R%S<6hYe*fnYOP86%?4gEcE(DTNnWr3ZkO2)L5Gd=5agj(d~_~zFL%an_!bXc zzx3jPf*=^K4QZt~DWQABB%VEsEqNL$Z|+NW5gkqBMRe>)Sb+M|bM$);+XN<^h0}=g zhA;c!^@FB%w=)vkmIprnOvM<&S z094GVfq8ffHV>}Vkle17+P63DikC8F9!X+~>~yB49eWvgI3S51&@0c5KI}znC{pm} zL*74o0!@5HNN)h&?w$LVfOWv5SA0hw;#0e>E52k^`zlPVFJ(nZ_GFJnG5-J$$XSuq z^Lg8TQ&B{h3S9cc7f6>qpV}%*p8^`iba7WcpDZ_8)@fS%+4}waV1p&-A^{CB|4AjA zQ!xgv0uTO)ZI%_~+vTYd1^Sz9xV_#fKwiQl& zn+iHGsPT%34C7R3L6?R4faUbRDDKr@vx&r(hhEFNggB>Tn7D#It1QPSm*p8oE$b?j z^#qrdhhEECMrCo(gg?{+zoD-N0nM$>lLUPYIUIGNd*oVOuy3B_>N@7tdKJb=hP1jb!v%}E z&qJ^HhXQ{b#P3^$ptc!PhcbB*QOpluxNQ)wkI9{q=E@QcO1YlV%l!@*&?sgXtZA*H z&Ov-uppW1YZQp9l*6B}Mq~Zx?FH+Dr$wM9*#Z1$_rg;H8_ydgWy+!rUefY!1(p->2@@kgti3Nv`9~0Q`#?OUS@%~QJe)rBdF}7 ziP~LCdPLy|Rq5Gt=&~Pc)hLB63DbScjYz#+F5NK~L4G+&4LRvV-Bh5Qbyk)U(`I)qhCt6>o2Hua1na7zU{$hPPIj4C5#2dPhlZiGkN+;mYd z+v672_p|grg8ms+r9AX1B1iA%;TA>k0%1~YJ>A?~wMOJA2G)%hYA|)Qh%a$>Bx(di zmFqfEtTuoHaBXFOL1p)1;W$r-8Pf$JIP)u6AUQ@@AvSw+{hSJr8rwfN!qyRH;#t(N zhbjtJl;^RQ`AuMcI5Y1`WilH2Wu(oR`4@(t4ZF=rl) zVopMf4T9GAn7S>c$AH4M!hl9GuK;GSEyY!*uIcho1lv*pjbbhXW^CJB*A&HfJ%HRA zGJ0|R{7Kx-dFU1YX5f!u&2??T>P6e5Tv=p|i8VJ5jbeW1@$7nJph!7}lz>Jt?*ZnM zytL#n<#_W^q<^hdjz639IUv7Ta zp+!^Ue5JUDHZWW|Zb?A7R11$^*is3cTk7X8I%O(s44!+c3k9BBy z!eP4T@|=#Z9IMh69EYX|rW(q#(5u)Sy>C5~XBRm1a#9Wfjbb{D^)b2W5J7L31+l^u ztl0)Mia8PWva4u5COg998L`*C3t>K*R`*F(B@C%! z#^5c2m{qx%&!xaFSB9^^+$a>_&YchWXM(+zfJOz<2}d%68K%@R_Lp&uN)SyP&?x3{ zz&tqFd39FIk{m&p4kE2_MpSFC`ZF>=B;A;jnirEZ{1M#fY z?KH57rA*b!tBZyocoBAZ>zG~Bh5Zn?*B=4KfNavOIO3&2@VY= zZb3&)i@eYkFDxN9;HVnPi+P+$b|z?l0hD|TCZQ7@Pcn&fFoWSoW#Sf0{9%sUno`0r zB*k@+VYjtzGdTX#uPX>+gIbDs#uf)_XUXpiJ4Coc#t-Jb8w=|R6 zQV{+x0)Bk_&B-UASNsN8A$ldLhcYZj3)h^|@4dUAGeLw^K%2W*!>Fd=i+4u6?cm{m{r=gGBPLVb8`#vhlX@+3iv>cvBvjdRNCas_8k&1p zA6=sa^dg*e7vNvccF57IFy$pjn2`LWOB;%b!+jFcLwZ-wp1jqQ=$a!UBLjUN8pYg~ zFe&T@VKG%TQ)!QbY_a=J=SuRDk&_R3XcXH;Nfdhb>u}?56njL6ewX=60P8SS_M177 zhgreR;8+MCl?8WUgwKNPRC*^n8z!|&`@D3#uvqolpYH}9*Ssg#vH%_T~0&7 zV&kLtIY_wLboEh?PmNKQi|@D6PB5+BJ72>0J#^Bj@ApbY9JY%I*e0&Bt3p7dRsH~&n?diV3VHCnXDgfb z+b0$6HoYxfcYdj>^0r`oTffS5oz%3-s(Hm#^Q&lC46#tIE&+~~3mo$<8+%7Td+3ux z54vu7rO$sq`zor-8?4=VXcV&>c8-b`{Y*BL}VyuI=M7f-&4FpJn&Iqm}*#e`Crx09IC-=cG`pZW95Q!lYC0gIHr zgA}2*9nM{rRm@Erx+)=;HU;0KJZsjDZA9gDbuf6bx%cS|L3XeK6&{_l1p%~BA`*s30Mu@jrpuV-@-#3nCSKLuID_t83$FGutCa0 zqnIOr*^G^xqnD_pX!GV@*W7#HfrPn`dr}7F;-Pm~X+XPp4sM5;#Y3*z%y-UQ@!a*J zZ3m_ni_gsV4-=kLO|u=Uzuf@hzD3?eHRXt>^e4X7(vh4FLmHfvisU6@-hpKNJ|{mZ z4QlO*@AEf`CzF(2ZsGgq_IdcqhY&vC;`KqHvz_JeF!(Su}_q^AWm$v8l=N)B0!>Cp(LO)e|<&O1!;LcBYJ z1I&OXOaA6s)?|mthurAyWX<~}va(rjf2XFS_RT}DJg|6mejdYc@ZE57BYhF1xtj^T*S?dNt zl2>)4W&h%N3#GFQ*$!wFvlQM(OU87@lZz%FQg~3zJT!`leG&6v zl2279S-?@_>6##*0p=q#KJfKKJ0-(Tg}swnD zixSFb9vbCy5NWyTwFN$D{ZIxDSNB9+c(bV?>V6Y7pJ~MIQcJw!p)oz4S*QYyZ$f2z zO=-aApMbDw%3@Q~1%==dv757dD*A7~R%=Uf0-}E{k5%cYOD6 zC#TFG1ejH<20`8b-51MRKDfUfI9^4uhtq@BmP^Z~)`~c8GeVqQ^QcXfpG+ zeW|TJO172GQe{oiEM$o}HBu4|SlccYYe>2QDMTV~hf9Rd3v8WJQJZH!Yq9H3eEFS~ z^E+TRkn?~;CLULR+@7iEfYc}7zCDPJO)g_aO#ezEt8F)}vT|0sY-ULuNfl=obLGjg zpJ-!?b}c5ZACof3PMxJ4JV$Jo(jg7BT);1PA+_{gWr_M_>>KldnY zux{!ZB)L)Y!*8U=_{xNc|{#QKIFIZ?S`k!aOQFVbRQ4hoQ@N$ z?wT^Ipg%qe%~Ps=>`4{byYHXhpWC!1WqNYEI=*&OG&`CLPLc2ebWpy;s*{ zx1d?COn-CqYxzlOAU}(_h%o7@w~FbNr02&#qIkH)r$PxMH7%f|zOmq+D2`EH!Y(Vn zD2cY)9Mt7olG;v%=?SD(_7Oe5IQsU$AqS_T!(REQ)9gbhQoEgKMVLlLd;5q}!|zYm zM($Sf`^s&`r=)4WBNBs0Y(B}Cm(0)7wDnQjt>>7sIVE&MG|dF*0yj!|R8|pH&!D=w zl_yy*(ZzKYm`(e!KN_vjGn^AP^t-0jt7I4U?A?8bPexS`?2zC!I)iMzI{Y0zO?|8mJZ3jBg3bL*_y4HWEpYDml8;FF-_%fI$DzSUMLBV z=-12J45?awCT7`+dlN(0Al_N7Dyf% zU|Re3Gwj>Eq>$OyY?z950oe?L;VYm~%-?~znktdiaN_RAc?gk5H^2;N6mtlP$(&2T zS(swSq8kVA`s3X@V9*Fg?|?=zcY>eYk_RFuD|VN6N<}-JaNXLa&mKXT!MGgIDCT@% zp2vJT%pTo~M-`qkl+Nk2y5gL18*@4$0gYlJLyS2;S$D~T5r<|Yf+Ub`&Zhnsx3^TrGEif+IEB zq@oj_d#rTrlT)lmxS4E@nGV?&7SE`zoZ%7f2o52A%vLuK4KUjQa}6F%s6M2E2cXt?xH}{F|9Q}PpD&tAn44H#^3W(|Cy1wW z5>x%*i|_lPPv32U*%&nb!GwTDTL%`bwqX?=qpv|1bZW)5))_Ts>491WG=ROF5YtMVg03F*^bCvm~b2sA%9Z=f2gg1P3Go%zy@%*P_+0!{d5- z5TEygPds#|O-)zEe96s}hX$C>5tW@-)i#}ODrXl@FRv`ACTOQIviqiIHFIcHKqJuW z7K6o^ECM(7NG9~?phqj(Z1YV&62Y3Je-O|Jrg;RI&!c+21(sQ@=O46UOlPG78o^w5 z3^3y!Weq1D@l=AwUC!=Lk@~*K%+EZ6I9aj3N{6*2GE~Snz-U=9_^!KzVnpH`njexJ z|9iyr=(S-TXjZp;kHs6`>SArHZM0d$U~LJG=-h2qf8^s_+oa+t3NDrp`c6Qjz#CDCMfAwR)EVecHJ$!^qXo{W1(*R1 zFs&)JNl;?JI7PUo@5h7NE*Y4L8Z_Mhx+$*}Tg9>M@(cQ}rS~T~0lnf+K{Ge8tNk=l zs>o7#(TS+~*lGXz&qp(f^ZKZrKko==6nT5nHA0?KLyM_NWSt=MBx$VWeJuLW8ic4<`;kSW)`@L2?8_&gQo$Lbvjbio&<|mBltPAhOkuSi^ zAY*LZmY}$Iw}+cu)+-fC?$#q;pU`qTMS*feUk}^$Y#WY3vCHX^Rh9Q@D)*Us`T?ov zk{-{$J$N^A5JJp=Myv8TFdHN>CGaoW^R+uhtx5qiFcAR_Fbe^9kZ06x!aE*%HBGEF zbp036pG0c84$)qcX|pV&JU9eLGJJ#Kpr;h{B*>H2s;159i&n>V3AZRwh=$`i^; zi=&cs@${0}>2jp7acpyUrc1`;g`!%MY|Bqdqea+B_piz3Q+HA30XZ-O8eq1CWZU7< zo*r4EP#;Y8;dAuofCiWc0?zL7j$=$WV9FwKwBoP&Z{KpvB-{5a{m~?X+7@;aLH7XA ze4wP18Zyt9S`r#2YZcjdVA7l&j7~JG1rBP@wWwxS?dJ>FdMZKt-PV4s8@{g$! zW*b!0XE&e$COipq9I3Ry-_AGeE>=fw zu9j#zJak1j`mx}tgYTMoNG@|GtG-I3n069A9%AiUMlrtz z4-Y`Kd`y-GouHBsql`HQ8tr2SG{9^^NNM8BSdvbYb6mu#t4`r2b^lGwMo``mCjBELgmyEfHC-_cU@{?}fD(az_5>HS1qs1&s&oZXtL62Iw+$8~@0Sz$GNcR36vC&SPx&*1>5&Dn7 zLokO6=oR15`${AIIr_&X!^(RO#;Lu)jSXlNvk44P;Kp*`2W=WZa@j+rUA8Ah3_DCf zLztUN|M#!`t1;ztJJ;X=@zN%)`a4bqSIF=9huD3kogL|RIaZj$<@<-(5ZnYvJT!t? zUmz>nsXl|s$78T-UrDGJjlTKI4-Y>CIh=##(tt)W?JoD>=nKAXa~Iq%6`geN(mnRu z?ij^fXAMwuYP9I}4{L^eb{*}v^{LbEoQ|;p`PkwsL8Dv{- zf46w<_^zCi+*YdyRgc&fIeK1H>PfTT(m!7tn#O_1*uLeVLAC5gJ_2_9yN3$XN7297 zhb9KqmyB#C!c$1&5y>q)`eJ5*!2;6@%;FJg=%l=$aBQ>KcnI?;4+3smM%?d+oo5R6 z2)Awm8pTBLgZV6pxpT?v^RHP?8^t%@_2||=e?-$g^LbM6BOlnR>4u+sq~hCR_Dd~m zU^)<5+MardyT1!NkdV1(@VTwN{cQk+9R%?cdU%Ihew6hRZOxmJkj$4nR6PZb@rZZP zD8yM*CL@@6XtXJ<^ZPvEkcJ)$>EjV~9e85Jl76%@#pVi&BORf|1zy+zPZ}xiW!}73 zPy4*_PIO}_So7C;G&)7Xu(jAz!G0XT@7UzhXfefwro{pLPAj=s^2^2bkdOjLqGJtr zG`{1(+Hlz-t=Vv<);jpixYlM6OX>!z`(AOZQDh2felT-4D(@ITvm~qqqk` zjc$PL@Fgwl>hXf}Hg@WUma)m~OdEFOoV)^h#kXgOs+0H)?iqFQcVCjRZP;?gx~BJ) z5YKIsm;sGq+O4}?c}#FTJazrJr7NG^BNhGea8cFUH{(8s05hOb%ts)Hw~~BvKOiGN z<&6zv-Z?9k3p1cmOvK-quVC+d*@~YZiJ794ZYbB+t$;=`TLSZgB%j=qnxvv9dOm*g zxh)V35!5}PQB0g)G3(eWI(h1r$kD?lv~8O{Di>xzqnPNCrZw8sue;3eq6PQ<*1q0a z^s9i+fJQN0sG5)I+VlWiGlrmzAj&MDQOw7{XWyhQN#32fjVPT4SsDyV<}o7WBOa!# zG;L_VrMSApP{=}7`vu=?d7q=agIU(~@Qz1piD?A-Psfw4y0qg0TOJh`Th8rZ2@5P~ z9rTJ1RWe`K*_OOc(UD}-n=x~=x6mkNUxG7zKwoIwrX6pFklnd{R`G}dA2820906e# z=4fqYr-CoL;&Y zC4i(Bb4vLei^VCB4zhAgIg5NFle>8VyCD?GDVu&&V~36_L#cv}T!%&uZ0hN_Zv&1A z9o##5xxg{bq1BCq-jRoxJTz#?=EQ_mpxPFtysui!4n?6C!z71dA_)hkb4lE!P98c# zST}OyPDei22NMuGhC2VixB0r&A^TvaCG+ciUv}6i%_j!}w6!v@{Mzt?7T~})jXJFr z+fOUVNiG8#mBC~ZX_l+E?sQ1HGjXLEDD(xop^`J(Fy|nxgC3<^2{<^EaWq=`1t>fo zQKOgd$n%&P!=H9csT{#jt+5%svU(WRfR9i584I(^fkJ&&Q- zC3C)&Mm`$Fya0Uu6ZF1>IaqEOO+=e?=yGgpbnS0oeEo(EXn;AAxJ$1C<{_Z>F?ms@ zv4l9cC(Ihu#qaLA5Y8&`VUhWhEuXU}@{6uzFKb{rK``f#Flw93JfXTcG9?u-x^AVH z6@|_z%OaO~il~X?Qm~4L&Kh)zLAbC~<{1L7Z5}5TwYDk;%X)f+E_r{Rp`8&U0Q(o4 ze9lB^C1u^72`s6@iN+qk(8aPyENqT&9A43&PzG6#F%4NcZI{7AqZ#93>)CE)i|vSd zjUpOSBy57)N3`LmfA{S0+`g&kg|DZ)y8rjHsDU^NC?AbBdT(G(v|eGlB{1=b7G3gr zuge~zIOMiBoHVjolNk;(4~=3D1LjxIfI=%77SC(LPu=I_KMqMn+uYIe)CX=ohcM&1 zI}rvnz_c5ebAfpRJ+hRO&F~JXsMj%rH%`AAbq9mg($4S&G>X}ZNKI?d`wFiE#}Vf& zy|wy;*><)89@1F-#*vDO3Uaj4be`4@ezfA1RbKGDmUlT+Wd|OQ9e)p&SHCj;cslp9 z%lzTHq~?@S9fHl*fJQJaPo_I`$ghLW0!n}wNHP9i9SRm;Fs#0yESQtAMc%79`h4o8CzaO>L`ti#qrcA&FS1sXp+IF## zg*{k*9NW6AwdnJ225r6>p?dRBeZL6;8m-ziG{L1TF1HvYi}ZT!6*U24+Lc;7Vyj5k zU9cBckTToy_~lp=|63h#Ep{*@8Z311eR8z$TK(!n`@VSE!BmwP({;gsMlm;&Wi)?S z^XQor_2ve@-+>N097vdlfW|jF0gYlJtAAqa-i?#NyoA=Lq0FIo?#*2k+J^If0(##X z$W}(|s8K+pNZyGcpi#`PAyynw$--nlJEfwvov&N@=X@+U2eGGe4R5sR-7h5(xOZ4 z{Pi#`)(!2fL|)Xe)RrTHYizgl%#AFLL_M}@x#WPCaZ6oL+ki$fdl36(0xOa04=TaI z<7O{A;}Iz4No+py&?x4SzNQQrQRd_WKUu2g$-8nIj3&XiF) z;kjkSRaJC_4V@+-CxY6~4O7}3s zPD}VGv}cAl&mMo#yWh3+^gEm_1T>0igI)qZRI?X6K5gE&(OtOCJn%yU8pX6OHP=0j zz*YN*ZfLyh{r3)|Bim1ZH|*@^PsSQjTz7YxoMZWSOhvK|Doa<0XG-G{6)&1Qv;WYs zv=r55>6cqSG|`IO9uKq2ukBgn2cXsQh*#NYr}%}-T8(_`s~ReHn?l=+d1z1-yB=>b zWAStWCLZxXLQ6fe=94j3F=iebVBP?T6~rQ)YN)8Bvzj&K<K+mHqIThr0m1gbP`Rl zts2lM=2FzSrwkZP%H1eqUx%`+JmUr zU?(e}k*cJhB+S}obXSQft}0JYqZD@jjD!cA@8wE~@9ETDZ>n!rl>!>Y{u%uCP4deZ zsP~p#@|}p^uUUhfu?T1svpK<;Z$R(!sb{r#AR0-SflUi&6tkExYnverXqK7 zpG#hH`2X^gI&q4f0lyzZW`aZC)&2pEVxrfXPnb_9PfcZV%z_ac zTJ=OX3D&&Ytlj<2uWzIELQW(yE@eJXs)z;v$w_`K7Mq#1ZDkjxh7wX0(;Zc`WL;=e zkZ>N7V0j`F<|+wjw1^K;v%Tq&g{hsp_g*jVJnF99sb;}uOhBWUBgs~nkCUycUM~;x zV+iwol;>+|d?iesSqR0vcfM z09$9h>g$pkEh#uPDv4sYl1&!S0CPX8O?nqtim@!uF7zMH#`NcjXBGb1fV?!`3WJw+ zIM+Q7jbiQz%p(}nX|&|Lh&ueSKc z@KBc|F;$l&Rx%Vh+k&Pkjln~)j}TBAX)Mg6fb_F>i&G!j(8FW9)A6&JEONJi63_tC zHY2hom`PlfTkwvDrrPipCIUk$L1pn-HCir0k0XtA^SQ(<=rcP}vBRY~Eb4Jhh<1Oa3aA4Zs*kf-#5lR)N8BlkO$7SO(Z{FQ^QKK=mf1=wQkS@@1ztdkC_-sU3BYpKQNq0y=!KE)PaZaN>1Zk|ccV>DuTN>*@OU4$EP+Xid}Fh!B7xkmN5@#v|4tF zZYt<~S7PO|Yf5GwNM{0vPU_Kh!TmUd8aU*t=LqxK`rtQGrkIO83mqSiNMoH2d~xif zH=o|grn7*)dD8cbl^z!HP)$*lVUR#Tqg6QwEsCYNtg3wXQo+*?p1pr6TD)6}^S&rS zRRZw@G>W+hm^jCrg(*WvwATp_AA42L<+(&2&?x3Tz^j+m-w$v9 zvDxITy=v8Z)v8rhtH^&oLb?e1T67)XwEqFX*IO3!4){g)k}s?Ue&O~I`-B z`HapCM(K3#!A9LW8fuDDig{o;U&0!049V!sU<~KkB=#IU8Z=AJOd}7v^%L8jf6S&l zO^}`b&R`6uKTfcVjDyiZNW=pNapjtI3-9|P2agPA2BSEupMave-?tR#lt2gb!+Ev4 zcHMvG8O?BJFotsqajq=l)M`(arC&X9>aX6w@@GzG24gtsbnBaUbagc?)|9P*Ho=|2 z8165U-psQ|$O0$1%ebxY9C|!GOSYvcgCWjssP@k?{CZA!5NkEH=s|a_g@O?mjYIcK zU<{`Zzg`#l$IOXDiMZoMJII8s4~|)^r#`5^T4CO!xlkmw(v7AG@5!jg@{zGAsw6E?-#@L2P4@M#4tvD7J(< zS{ltu*o*6(f4Sn+S34$hKNEM|g0c$;&xaM037Zaww<))tC$~2D+dgnG73H@Zn5L<# zuqo+PADwzhw~`ZWV(F1|p%}-sHLNvBm%y-lp0TLQZ=N%1hOt9mOc2*VDKs@-0&n;! zIJdWW`zSQs>lcdS;P{}@!}3)178GEuHi|j>!dmbJt@e`^BJ=P0+UVVHT!!7_S8egt z%T9U)m6E`H-(z3=W~*7cX(0U&xnrTT1jgu$f=}-W5oN{k z7&)Z#H-EL;r)bbVjLyQ=4Blw=E@yiX-axaR6&iKlTpYZ55`4&L&R`7Zmng5X!rRzK zn<1VxWyKE$`hXLvL7JD$_}y;frE4)uZ7WUcpAe&xrD-xcgE5@Gl;DVV@kvQ4 zJ1uuFx|8gAuoliFS$u@fCotB+Y{G;%Ey4>UlAS0N2zy!m69uwGJ1?tlUxZ3`y=R~+ zB_CQtT+R%IQIp_JEEOArbcG$sXwJIRfG~qGoSOj@J8oG0xU+XbAFnQ`u^B#W%{8n% zpopVNiU+9cV^D;mdp_1ydvQLytNTpc8PeA$p1%Z#HSG^!|BFaF!rQo})9ad>Oy`u| z{hPrU&TC2gcknq@*Ck-Uv|0?=o z+Rv@Y*sSNT0Bc7ZCEr*ZeoymZqA6DbqwKWM74YDuw}w;)@M8l`>#DxE1!x~2F<)oy zNTuZt5#5-|miVwB#8%+kQGz8hNPU?lgH&=CgQso^#Mi)2(_lRgVtu_O;U^Ee$VnC> zQbms2tA)xE7~=FBz8An+)kDQpiVdCOsu4}NH2?&ivFFN@klx#>&AlriQ3=KEyww^a z!jJj1nK?SwCSr?*CMa=$My!&@E9VQ&5*TC8#;`;8F>*i6%bI#xXy3)RjNzO>oNo!vA`KW6BAuB&3Y`fIasG_x{=~evu4L9Yn^w{0N7Dq(>%#2x1gtr-tNg*(6S!Xq?r}p{Q7(??jcbc61$0 zdw0i&jl1^Zju5Ml=9f`tJ@zG|??)5p@k8j8R7RDZr?{T1XYMud0HY^!O!w-wCz(6a zSLm=#R9?>le5{lyt0Cg;ssP_)-IzrE;+-c5b} z)i-+%c&}xez34)*&?gb1m!A$Af1Kz+DR4Y2AoVsT%qI34@-8~fNAV_lK@;BM!7eTDHjMyLx-eCbr_L!?PG zE8Spt5?PT>f-=)(%YM|n0z*HdArv9UK7|H{sDp4aGz>`OQFOU*3F^;tuTgMTxH5g6 z41`UvXiwb6(#Il*O-$}#YGv)*f0%dfoZmapH4$mV`F+@>OQ+lc8P~VN@eAx|Sy1ii z#+$El%RBYv;y4gk7lV$B-E=>`E;-lp>LYVBkXFg(l-4wS>3&1+HL4xe73X6Qq*peG zuABQlK|o{|Hr1rEg$9zQl;g#2oii_V*6B&cY= z6JcH$%|=a>ek0~qq2mf`kd4ljz!=UGi1RqYr#Qs|u=c9OM4B8|&R`7ZT;klNNT7`C+fnF@EI0VCOmCU=*d33E1Vvynp&DFaE-Y;hp)}X8cS~M8*zF&0y>^ zeQ@|jV{30c-{yEuwZVP_4f2P zs+6b*NIUP(Z~ov3!D`hySFjcxAKqUg-YK6-uEMoVA*1w@xQpbW-D$Aejg<2+F-)GwNY zthpW7d@8LB#!ypSZibK>1!a-M!W%y}csT@B%e`&}V>o|FoLtF+mBj!{^AMHShivkz zyT7y=IC+CzgY+{P!%5dwA2GLgmbfJIC8_gvZ`{5tI<;NQ#M#_e$490N2L07nq+{4g6u)voy80hobloBo)ySlY<@Bf%~ z^94U*BN^ctj5V?);}f0|Z}86ozfoJIul9xe7q9u9Ph@%xB~h-s$e0Y?@ZSae^%h(s zf!imV3iOc)H(z-F*=xb>Y~HJx--@p?|NP(D+^cYH4Z0}}!(8V*HD#~m=hvety$-a*$Xy|;@_b*{CPbv?E3;p*55GWrio%b8*hSnwu@AS+bo5Wa^aABqCN{F8AL;f9eI1?iF#PsU44DQRgE+SZ zs1ejL5gG6Wy**^n3U)JyFAgu+aI_&R0j^vNhD%Xxk1`$MM9WM|ee~{5g=zcqKRC~W z!lzY4qYdgWbYOw%prdF1_~tj@BXiZvU?85gXa9d=Whipl-!N0So{Bfz8=lMok>}}R z*QS!)d{{7GFZ7T8Rw1AJ!7;&w1BlAQ98$i|`I%U)`wJ4VP~0fRrYv7b#pS8ZaI!w$ zX%(%viAVTnYI$kh{>Z9gd_Ct)N*KZOP!S#vDjKr8L z;n9Vipabp{iAo}fIpJe=-DL8pGuHSK%Jb4&U?Oz=>`Cg(3-EBX+Uz*8Y=kx2G-}Siw$_H>Yxh@5KjpL8#omvnNxdGvGluHIr{N6yP(yFJE>1l67Ne~tcz5BggTEX<1n_ZM$21A^F*tm+rW2jLX z1(Pe@^#Ohax{nL9xR%(@Crd$;d}r2wLacDfpwlrmLPfw=Qs*S(tz02z$oWGTt|kZO-? zNJ;T#Ji@`7=E|FRSi^e?@opIhRhEBwA1QqspOmYt5>O&YBj9uru+^+3i;%UE$j<9Y zi9vjz{w)N0_Z?E4^2{8Byle)eSzP2<24f`Cv$`_~pW+lbJf`t2FB#8pW-x|x6>*+j z#A$}5@p~;FF)|p#NmuO7EaDUz;OulEtO?ExhB&>mqw{i~6P(e(D0!9Ny>RKR^>?2D zKf~Os1jguWp#wdw=#*_FG<#IFin@`(7|u50dAV z9KH5E(kc8b*dO`1>QN2`LnXuH$AWy^!zt4GMNbYt@olm-`1WX7CVqt>x&E}J-`ePX zbvlzaczvn4TR;O?5T|zflO;Zt1=Z*Z)7hm+XiaNU3n$zEk--?wgQ)r|@qnnJd(}nu zBwv3H)h?X{XE27-Cv?XN&d4iEtUk2;%#{a{pQ_eh0BU!>;4gtU{N8PD)jp@AAGfK7 z3?aympNhmAMz=6)EP*lfbI6kGg#4lwHQ+o=NG^dfoQsL`bi$`z%MyXxjlTAlJCMM{ z(N(saT0{mzoId{__tvHB1p<$oIdrc$M^X=eJiSKv@3B1wIdsW@55^lzoIXzoYk*kAo1SX%VsdET75AV^YB;N>>^+DmL z)?3MQK5U~`<;by1K6cyT(|la-d9r|nXMMC4M=c#ya~FuLBOfvt>z6-gb)v{R!mA^- zQY*OM7j6==E`c$eQFf`UZBvZilv8sT7U|4jOul56`;yQZc_n6z5-+!W^4L?3er6i@ zzgi}QB{0Ovy_3m)&vbvN=Cp!K@1dSp#Gk=iw<3IsU&ui8fB4>iUUUBetX+`Zglsh9?%U5!dHM7< zH&U9NN^9gJ6iUGX_J`M8bm*Kx^SpMlTdTN#u|6!!;QQPAJA3>5;rF=1G#e~x?r)-1 zB{0^lpP&)E%O3OfHoVuWeTrs(q^Sl%lC@S+M3ZF@0@t@?+5nds})t8#vr%0#%WH5$v9njSW&+A^&j-CIhQpzVu5C<2+rc+Wg7(>gG zDQ>T#cBo=jKJ)k{JDop@-Oq4lFou)oH{8BOoK|w?lPeC{@!pFmrrnAx&0q}Y?v%gx zOKION$$Q~WS01~=jl}s}5oZQNoV!y(+$gKn9pd&&;Z-s$*W3i*X9qV>tcBun!OCQu6aNlzUM7r}+VxD;V3c~-N$Ll;BN+F@$4@fB)Z9>Tufb?D7(@Fo8T}qdvzDZI zzJ_F0)|2sPl{!AgV!5=~eg?1jSFHirdb4;GlKH#C@16GZ%RhIR4;pSF=r!!-@OCK! zJq3IJdwW>1BW*1E*!b>Co$mn*4PH$OUgD;K{~X?b#M6tXfbW^J%wUMq2y$N&59EV& zKEf@3kfqeyN9`;y$iw?9yPqj)r3Ocp&%Sop%1hf0Z&UD}lc@nb%G4Z(?u>BlGKr5M zB2vtp<qp#+9F=`I~}xT;qX-c(gz%)LhVlx78MNX~fU ziq_JU#gcdCf%c#ekp^S2!WbXkgqU-h&35#q=1yO)eJPCLe3v*oD2ovu>9><3zy2AaF ztSHT#4Tvp)F`OSK&Yu;Xffx|Z*y+AB8>)(in7|m$J~Ch=_QKh)aLAHlI{Ze;C}$A` zB4ZgPUV|NH&OU>&ChlMpqZ^wzt{1EHQuj6X+s7niBDg0oZZKgH7& zdyxc=M&Vx0wcntnq~Xbb(5m|wa7E3g@j7fh}I+fY}OmL zKd(5{UnA2uud{zrgNqaYjdh=kPv7#m?dGylV#f0fhNx-N)xq=`egsTyTKpD@z-XQ8 zCN`bHL~_9tY`g8~N6oKA?KpnCpaV7C&aCzqoU>>`5-3xQ)?s&5bGan&)}o)IE*#|L zQj{ak7K{5qNxp!;dKr3MA2tg|NV;&Kp}VKkE$M5#OG9W&zGYiJ2D_dd3#0d^`OZgJ zbVI2$Ie~^EJvxHLu*J${hyjjobKH~JVAB@LNrqyW4;p;4+=t-qV4A8$iYfZ*Y2}JL z=#0LYql~9aEZLzi>i44AOK%g6cT0t`*g;7K;$jnh$@?c-mhAS=`)31zz8G64LE80A zS(o?Np84FI7n>I9kekpKn|MUtiYGv#N?&q+sea!E0BidJbhbXMd-iNzRI#}GgpSHI zzq`MvN)4&;V810WhChbwExy2me^lh;KJF=$(?6)N0u`edOXgo`cz>8ll!*? z@Fbp0ot{mXMJyxRbhZ{LRxfPzVZ?vLoO#3iAdRCH+CI~^R-KQZ z>YJgh!~D>Q#{7*beAp>(dqH7D*cJwfIDc2)X%iNAxqw7wh_6d>O8FF4EGQXGRYjww z3rLda)GoO1Lu=|S=)w`ra+nQ%@_R+v1Zx6gSp6ErQo`rK2u`%OgxfD_t`n>ROJaX* zqNOl~vm2o57b3iQKJh>65U6H0oWU5*KKg~AfKha2(E8Fubhb>SFS+P4_N-5I2?ul0 zX=L1Re$p$gFQfI{L7vH;c&dJwpyCdD;UuyTY8erJP@<=zU|3Sn7oGG)en>|ULMck) z-$k7KUXsX}UNN}zBnUAQB77?IoKfq^!5E_+^CgBn7ifd-G^$aH+lS=v5ynBR42tYG z_FRCl!%qAs@Y)d7o7(9Mn06idmAI&M^vWpy$DGPbJ$?_SXkGOEaGcdmK!1G%CRjSF zI%LTSJlI{%)hxSXUQ}O9u>Qc^kXbk!c|u2}!HA{niMMHDa2br@J%@yE)<1ybg+Scs zo-a$}JZdcjr5dS*Rsv(FeHGu}C8CO{!?DXFYKpT2 z#&C9n5pD~a1@v)7z(=s0)fyG8hBph*X$g$s`~lT}M{x*6-`~*pa2a_CjNm4AxhqLws{I*^ zwZLRqeH3g53V3u!AL&zC8x&mz=NB}M`%08W9DOnvYlVDpx2~6O>eWrY5vS>@!Wd3& zGN=#6SPilpNVQM&g$~z?!)FG6P-KpRnH;SN?$fDngWXEU`f?E+^N)3h%7I@r7(-qa zo5|1mQ37L){5KmxEdsyUzsu54^ajPxlfY-aO(ZaE;z@e>r}zjp@M63b?0k2KD;zH^*MLA|L#Jdsjy8Y-^YCoD^6tN1GwEwf_r+zNr~P*9El>AA+F?}nq!BQOVOKoPmnfAi zN2h~txKeJrh%-8c42C$VLDiW1PeW~+`{~hZOA$w$8H~~C_pgTer0+Y`BV@3TY^xkR zbH`8rVM-t9%<_|(y$aiyW*}DqmNbM~Hy(7}vS~LR&{p}xi`PB1>+eo(Q+x3VRL^Cp zul(t@%7HH)d+~qt;AY2cDOTb-tQ>AW`PxOR+GqLvnrFym6x_Fpo>%e**yeK2PUpN= zTm2KCLi{vnbJGd$_gkdnM+R>Ue}wQ~75v^L_^@o1Z3#1mm%tcKmSuLEWB6`x0iVBm z;fl$Jj|QhkJuvzUW0?@Xy66Hj7~&jCJv)${+Z=De8FSMuT4hzw2SbP=p{N*^{?jc? zZjyi$gQb^rZ}{upZhW&FLn+-y@IbQ^rhnKt5xoS+>?i1#(RIdcKE;fD26Uz5GwU6z zQ~Onn_A`22S39b2R(cl5r4+@EB-ekgx3dWhoA@0W=&P&zK^W+*z+1OjqIe)&w{m@e zXNblfaJY~RH#&FqF5ngkbf@H7Ys-gscZ*U*coT%)vS4lb^JYWXOtFpZ8*78#xStjc zLp&fPdDZwA_Z;~x=DV|TCW9f)SBTClR)A;*t7ldXuv5ouW%26-jUiw>KP0vHuvL}iy$JSj0VkW z*PNr0r@kqJF`N}@=OV(VeHZ3qVPmJ^e>|#w!rjdbErTIWKT7#ae)lT&UA#H2k=68g zlfGv#hBJ{+6yYU&!Aj1|18#oq zn#IH!%5nK!gnWd{b8wg6$}_I`#hKGld~63{&lsZQ!}6Pz?~dE;wR3NKJnq3|y45Ws znJo#K;sOzz44ZjrdzcAdx0!wwx?eHpe6cz^cc3c?>R zG{<8!*;LkXDh_VOZi&>BOoH2j9wr^(Oo*a&@3gut1*C`tQMq`Uhg#jg78|m8it4WQ z^;Sr%KB%u_(HtM06eF9@hPX|NlLqjbv*~}aHXcG;6;TY`sPFmd0 z-lv&mtqSgQpAVy{2A<;o{a;020KPYs#4i&E@E-gqm)eShNKzlxiPte2)`#E_cb5Dz zZt6?T{of+h+Pab99(aF{2L&Wjrw!QrQmI>Y2^!!s18O%wa8c`|KiHa z<~`bNdg|ZUq7>vcs053t-Lvnnna}!>$BR13F4wxm9^fg9s+`%3vP9+`E#iS9L$r^U znyIh7PH@!%#IT3xyYm=*cX>f49KdlIP)JkG7t&{9G^k8xN zaoiMMVH3LY%A9@)jN#;3ojXa?B!=9A6HZ56OA|3>GRiI3B&}-U=8FQy-kOnXH;l5& z!NL0A`5m*m=QFuq{3OxIntMVD zbnaKgnJIA-Ix`r<=|{Q!i#T(i@ImNKK;fyL z+11K`liL+i#mxl6p`Qal^#mA(PS$v6a23`sC8i0^ z490L$Vcg>%E}c|=_gE2U!3#drtnV3&(fMuQOtq)Kb58eyj;PD24jKn_>roSqWH8ne z_0&DNUh0*^COGw{ESRhMi>ZC!kfV&69_Q>SJ4+E zuoG;l4_Y9l4xh%)a>iC7xnv7{6Vilb2+{OKNsHQ{TrD9j12NM4vA+-}MU4o#uoD(`|JIs&6B~Z?CBKJAcanx1gtl&TpDnsA;W<{j z^y(uc{!VJu4~6(RkeVY9#eT={!SbO92NU;{SZWD8@%vH~7Xef!WHA@6orzf^g};yt z42jQJeO%y^)TC<{65^aBGjsQ7kxKN^|5fzm`Y^feh9l6)Z4qPQk@NE1S)eXNfye# zibxn99wSx|wN%9(d^@%Z9ajXAuTz;H-yEdiXgp=Srq*$hs)_S@EV;T#)sDq0aB&4L z3Bh$ays?NeS+U5r_9v5X)fdV57isQxHqA#Yzz~zCucG4^HDeYmbL+V|*;WKGy*FCj z^P+hp$#OnyWU-PA+Alxn>xQPF(I`=KcG9EOT)gmXZ?0ZHLY?I>yE~wK}Q5rBDtEILYVa4vJA#>lG8fXiA?zOF=zbmGk=NGPLE%5;k1=2 zr^C%>R;*ec9Q2enmsy+^5vcSYy5^>Nw;$CcN2~iCDj)yIR^OWX=RldK=l3V`zvp)dmZv|)6U=3)4RXozB+7X+T9=y|9`NFo-mOb$7 z3z1*6JNJS#8zpJGvh>nN(~c4t>q^x5G~a1ckh`upsC)0z9`PG`rlMtkNCFpAuj_`F-fnYMVQdVG zY~;VO^`z;yu9MQw-b{U16kLDwyGEwItap(uvAaBmQ46+!z&$6FMfJ^`mN8^8W0H`h zjPPQ~i7Us=hY&ohy#&T^a;D*i(1)c=HU_FwvSeyh-weiZ`a?1qoys^XU0=)Dlff9y z^GWA=^7t5YG-|Eo(U5c@fiaw4BhD;QB-JKxF^G)z`@<}=#q|uvaK=C&wGUHT3Bp+x zVpLJ?MyD|L)h28DI!1-9DkGz*$7PGu6WUOuC#jn>o55IDs>~kvi#gK(#AuKUm=8c# zvcV;TF`NsCGh?}keOOy%pFbYi`}y6c+CGb_pJ#LxS(U*XeikXwxb0Am;C$H7YA?2g zbRJzKA7PZoIIIJ*#%j?Ah=daRwr0n(*`S@#Gs~EXy{Cc_+Z^4q(v=bz!%5C7t0I|N zVyAN9()+Gn@hDi87r?cMTllmNnk3D}kV58CAnlv$PTFDrCyt1k6fe?E$3JnGI18rN zZZ{eKG8pR$Z`*ZKNJBc?RYu!FU@dpk8I0lNRlM#{!5RCdwqx7sV7$e?XE25{&X3Z* zn}tZo(Cm+=e zX9i<9e?&Tu<-n185L0;(1T(TB$}SErFbjn-oDUJ_zct5c0~R+Ltp{fUV>nk5=d|WH z%``bJS`W?y#&GiTwEC8|`huxRV_wFzP+!V4RbdP}7s;KUCZt+r!SbcwTrv2p_trpK zD@0SHbI4!}=h?)0B;iwwGR@xW{I~Dsn4mjZoJLTN3Q%YyBMlT$oKAxBSzh@{qMc=@nrhkUx>Y8q+y94 zeW|%iW!6;!!>)Lo{tWKR!m0>wQS^ z|Gw92TQM{R4L+uf;!2b3y2ZNk^wPL^D_LW~e=0_Qf;j9v8SPG4F3OsgZ%fue_FmqX? Lyv^YE*yH~JxRaDs literal 0 HcmV?d00001 From 3b9aa3840be567151013d75648ba4e36271a3a4b Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 26 Jan 2018 11:23:31 -0800 Subject: [PATCH 095/519] Upgrade to 0.29.0 of google-cloud-bigquery (#112) --- packages/pandas-gbq/pandas_gbq/gbq.py | 5 ++--- packages/pandas-gbq/setup.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 67d5ea515ee0..514afa4acdd5 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -21,9 +21,8 @@ def _check_google_client_version(): except ImportError: raise ImportError('Could not import pkg_resources (setuptools).') - # Version 0.28.0 includes many changes compared to previous versions # https://github.com/GoogleCloudPlatform/google-cloud-python/blob/master/bigquery/CHANGELOG.md - bigquery_client_minimum_version = '0.28.0' + bigquery_client_minimum_version = '0.29.0' _BIGQUERY_CLIENT_VERSION = pkg_resources.get_distribution( 'google-cloud-bigquery').version @@ -1241,7 +1240,7 @@ def tables(self, dataset_id): table_list = [] try: - table_response = self.client.list_dataset_tables( + table_response = self.client.list_tables( self.client.dataset(dataset_id)) for row in table_response: diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index 86a40c5eb14d..3aa4b1706c04 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -21,7 +21,7 @@ def readme(): 'pandas', 'google-auth>=1.0.0', 'google-auth-oauthlib>=0.0.1', - 'google-cloud-bigquery>=0.28.0', + 'google-cloud-bigquery>=0.29.0', ] From 57962af343c56cb4b89555c31988784124e7cfc4 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Sun, 28 Jan 2018 14:22:01 -0800 Subject: [PATCH 096/519] ENH: Add table_schema parameter for user-defined BigQuery schema (#46) * ENH: Add table_schema parameter for user-defined BigQuery schema (#46) * remove unsupported gbq exception and replace with a generic one * fix versionadded for to_gbq table_schema parameter * fix test id numbering * fix tests by using pytest raise asserts --- packages/pandas-gbq/docs/source/changelog.rst | 1 + packages/pandas-gbq/pandas_gbq/gbq.py | 14 +++++- .../pandas-gbq/pandas_gbq/tests/test_gbq.py | 44 ++++++++++++++++++- 3 files changed, 56 insertions(+), 3 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 5d1bb98b93d7..c1e09e9d0887 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -6,6 +6,7 @@ Changelog ------------------ - Fix an issue where Unicode couldn't be uploaded in Python 2 (:issue:`93`) +- Add support for a passed schema in :func:``to_gbq`` instead inferring the schema from the passed ``DataFrame`` with ``DataFrame.dtypes`` (:issue:`46`) 0.3.0 / 2018-01-03 diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 514afa4acdd5..69f9f29f6878 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -890,7 +890,7 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, def to_gbq(dataframe, destination_table, project_id, chunksize=10000, verbose=True, reauth=False, if_exists='fail', private_key=None, - auth_local_webserver=False): + auth_local_webserver=False, table_schema=None): """Write a DataFrame to a Google BigQuery table. The main method a user calls to export pandas DataFrame contents to @@ -948,6 +948,13 @@ def to_gbq(dataframe, destination_table, project_id, chunksize=10000, .. [console flow] http://google-auth-oauthlib.readthedocs.io/en/latest/reference/google_auth_oauthlib.flow.html#google_auth_oauthlib.flow.InstalledAppFlow.run_console .. versionadded:: 0.2.0 + table_schema : list of dicts + List of BigQuery table fields to which according DataFrame columns + conform to, e.g. `[{'name': 'col1', 'type': 'STRING'},...]`. If + schema is not provided, it will be generated according to dtypes + of DataFrame columns. See BigQuery API documentation on available + names of a field. + .. versionadded:: 0.3.1 """ _test_google_api_imports() @@ -967,7 +974,10 @@ def to_gbq(dataframe, destination_table, project_id, chunksize=10000, table = _Table(project_id, dataset_id, reauth=reauth, private_key=private_key) - table_schema = _generate_bq_schema(dataframe) + if not table_schema: + table_schema = _generate_bq_schema(dataframe) + else: + table_schema = dict(fields=table_schema) # If table exists, check if_exists parameter if table.exists(table_id): diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py index 78928a60f0c3..f4f731b1144c 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -174,7 +174,7 @@ def make_mixed_dataframe_v2(test_size): def test_generate_bq_schema_deprecated(): # 11121 Deprecation of generate_bq_schema - with tm.assert_produces_warning(FutureWarning): + with pytest.warns(FutureWarning): df = make_mixed_dataframe_v2(10) gbq.generate_bq_schema(df) @@ -1422,6 +1422,48 @@ def test_schema_is_subset_fails_if_not_subset(self): assert self.sut.schema_is_subset( dataset, table_name, tested_schema) is False + def test_upload_data_with_valid_user_schema(self): + # Issue #46; tests test scenarios with user-provided + # schemas + df = tm.makeMixedDataFrame() + test_id = "18" + test_schema = [{'name': 'A', 'type': 'FLOAT'}, + {'name': 'B', 'type': 'FLOAT'}, + {'name': 'C', 'type': 'STRING'}, + {'name': 'D', 'type': 'TIMESTAMP'}] + destination_table = self.destination_table + test_id + gbq.to_gbq(df, destination_table, _get_project_id(), + private_key=_get_private_key_path(), + table_schema=test_schema) + dataset, table = destination_table.split('.') + assert self.table.verify_schema(dataset, table, + dict(fields=test_schema)) + + def test_upload_data_with_invalid_user_schema_raises_error(self): + df = tm.makeMixedDataFrame() + test_id = "19" + test_schema = [{'name': 'A', 'type': 'FLOAT'}, + {'name': 'B', 'type': 'FLOAT'}, + {'name': 'C', 'type': 'FLOAT'}, + {'name': 'D', 'type': 'FLOAT'}] + destination_table = self.destination_table + test_id + with pytest.raises(gbq.GenericGBQException): + gbq.to_gbq(df, destination_table, _get_project_id(), + private_key=_get_private_key_path(), + table_schema=test_schema) + + def test_upload_data_with_missing_schema_fields_raises_error(self): + df = tm.makeMixedDataFrame() + test_id = "20" + test_schema = [{'name': 'A', 'type': 'FLOAT'}, + {'name': 'B', 'type': 'FLOAT'}, + {'name': 'C', 'type': 'FLOAT'}] + destination_table = self.destination_table + test_id + with pytest.raises(gbq.GenericGBQException): + gbq.to_gbq(df, destination_table, _get_project_id(), + private_key=_get_private_key_path(), + table_schema=test_schema) + def test_list_dataset(self): dataset_id = self.dataset_prefix + "1" assert dataset_id in self.dataset.datasets() From 566b3079f51546734044e485885ac45ba8dcad99 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+maxim-lian@users.noreply.github.com> Date: Sun, 28 Jan 2018 17:46:43 -0500 Subject: [PATCH 097/519] disable codecov (#113) --- packages/pandas-gbq/codecov.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/pandas-gbq/codecov.yml b/packages/pandas-gbq/codecov.yml index 62aff1706008..89cb6807a59d 100644 --- a/packages/pandas-gbq/codecov.yml +++ b/packages/pandas-gbq/codecov.yml @@ -3,7 +3,9 @@ coverage: project: default: target: '0' + enabled: no patch: default: + enabled: no target: '50' branches: null From 78947b75c306947ceb3126bb5ee6bf26eb970be0 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 12 Feb 2018 11:30:18 -0800 Subject: [PATCH 098/519] BUG: Fix uploading of dataframes containing int64 and float64 columns (#117) * BUG: Fix uploading of dataframes containing int64 and float64 columns Fixes #116 and #96 by loading data in CSV chunks. * ENH: allow chunksize=None to disable chunking in to_gbq() Also, fixes lint errors. * TST: update min g-c-bq lib to 0.29.0 in CI * BUG: pass schema to load job for to_gbq * Generate schema if needed for table creation. * Restore _generate_bq_schema, as it is used in tests. * Add fixes to changelog. --- .../pandas-gbq/ci/requirements-3.5-0.18.1.pip | 2 +- packages/pandas-gbq/docs/source/changelog.rst | 3 +- packages/pandas-gbq/pandas_gbq/_load.py | 74 +++++++++++++++ packages/pandas-gbq/pandas_gbq/_schema.py | 29 ++++++ packages/pandas-gbq/pandas_gbq/gbq.py | 95 +++++++------------ .../pandas-gbq/pandas_gbq/tests/test__load.py | 40 ++++++++ .../pandas_gbq/tests/test__schema.py | 55 +++++++++++ .../pandas-gbq/pandas_gbq/tests/test_gbq.py | 23 +++++ 8 files changed, 256 insertions(+), 65 deletions(-) create mode 100644 packages/pandas-gbq/pandas_gbq/_load.py create mode 100644 packages/pandas-gbq/pandas_gbq/_schema.py create mode 100644 packages/pandas-gbq/pandas_gbq/tests/test__load.py create mode 100644 packages/pandas-gbq/pandas_gbq/tests/test__schema.py diff --git a/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip b/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip index 18369345e8a4..ec27d3cc89f3 100644 --- a/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip +++ b/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip @@ -1,4 +1,4 @@ google-auth==1.0.0 google-auth-oauthlib==0.0.1 mock -google-cloud-bigquery==0.28.0 +google-cloud-bigquery==0.29.0 diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index c1e09e9d0887..c3d0aa743acd 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -7,7 +7,8 @@ Changelog - Fix an issue where Unicode couldn't be uploaded in Python 2 (:issue:`93`) - Add support for a passed schema in :func:``to_gbq`` instead inferring the schema from the passed ``DataFrame`` with ``DataFrame.dtypes`` (:issue:`46`) - +- Fix an issue where a dataframe containing both integer and floating point columns could not be uploaded with ``to_gbq`` (:issue:`116`) +- ``to_gbq`` now uses ``to_csv`` to avoid manually looping over rows in a dataframe (should result in faster table uploads) (:issue:`96`) 0.3.0 / 2018-01-03 ------------------ diff --git a/packages/pandas-gbq/pandas_gbq/_load.py b/packages/pandas-gbq/pandas_gbq/_load.py new file mode 100644 index 000000000000..45dfec586242 --- /dev/null +++ b/packages/pandas-gbq/pandas_gbq/_load.py @@ -0,0 +1,74 @@ +"""Helper methods for loading data into BigQuery""" + +from google.cloud import bigquery +import six + +from pandas_gbq import _schema + + +def encode_chunk(dataframe): + """Return a file-like object of CSV-encoded rows. + + Args: + dataframe (pandas.DataFrame): A chunk of a dataframe to encode + """ + csv_buffer = six.StringIO() + dataframe.to_csv( + csv_buffer, index=False, header=False, encoding='utf-8', + date_format='%Y-%m-%d %H:%M') + + # Convert to a BytesIO buffer so that unicode text is properly handled. + # See: https://github.com/pydata/pandas-gbq/issues/106 + body = csv_buffer.getvalue() + if isinstance(body, bytes): + body = body.decode('utf-8') + body = body.encode('utf-8') + return six.BytesIO(body) + + +def encode_chunks(dataframe, chunksize=None): + dataframe = dataframe.reset_index(drop=True) + if chunksize is None: + yield 0, encode_chunk(dataframe) + return + + remaining_rows = len(dataframe) + total_rows = remaining_rows + start_index = 0 + while start_index < total_rows: + end_index = start_index + chunksize + chunk_buffer = encode_chunk(dataframe[start_index:end_index]) + start_index += chunksize + remaining_rows = max(0, remaining_rows - chunksize) + yield remaining_rows, chunk_buffer + + +def load_chunks( + client, dataframe, dataset_id, table_id, chunksize=None, schema=None): + destination_table = client.dataset(dataset_id).table(table_id) + job_config = bigquery.LoadJobConfig() + job_config.write_disposition = 'WRITE_APPEND' + job_config.source_format = 'CSV' + + if schema is None: + schema = _schema.generate_bq_schema(dataframe) + + # Manually create the schema objects, adding NULLABLE mode + # as a workaround for + # https://github.com/GoogleCloudPlatform/google-cloud-python/issues/4456 + for field in schema['fields']: + if 'mode' not in field: + field['mode'] = 'NULLABLE' + + job_config.schema = [ + bigquery.SchemaField.from_api_repr(field) + for field in schema['fields'] + ] + + chunks = encode_chunks(dataframe, chunksize=chunksize) + for remaining_rows, chunk_buffer in chunks: + yield remaining_rows + client.load_table_from_file( + chunk_buffer, + destination_table, + job_config=job_config).result() diff --git a/packages/pandas-gbq/pandas_gbq/_schema.py b/packages/pandas-gbq/pandas_gbq/_schema.py new file mode 100644 index 000000000000..25e3ca9ba358 --- /dev/null +++ b/packages/pandas-gbq/pandas_gbq/_schema.py @@ -0,0 +1,29 @@ +"""Helper methods for BigQuery schemas""" + + +def generate_bq_schema(dataframe, default_type='STRING'): + """Given a passed dataframe, generate the associated Google BigQuery schema. + + Arguments: + dataframe (pandas.DataFrame): D + default_type : string + The default big query type in case the type of the column + does not exist in the schema. + """ + + type_mapping = { + 'i': 'INTEGER', + 'b': 'BOOLEAN', + 'f': 'FLOAT', + 'O': 'STRING', + 'S': 'STRING', + 'U': 'STRING', + 'M': 'TIMESTAMP' + } + + fields = [] + for column_name, dtype in dataframe.dtypes.iteritems(): + fields.append({'name': column_name, + 'type': type_mapping.get(dtype.kind, default_type)}) + + return {'fields': fields} diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 69f9f29f6878..8ccd2511dddf 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -556,45 +556,22 @@ def run_query(self, query, **kwargs): return schema, result_rows - def load_data(self, dataframe, dataset_id, table_id, chunksize): - from google.cloud.bigquery import LoadJobConfig - from six import BytesIO - - destination_table = self.client.dataset(dataset_id).table(table_id) - job_config = LoadJobConfig() - job_config.write_disposition = 'WRITE_APPEND' - job_config.source_format = 'NEWLINE_DELIMITED_JSON' - rows = [] - remaining_rows = len(dataframe) - - total_rows = remaining_rows - self._print("\n\n") + def load_data( + self, dataframe, dataset_id, table_id, chunksize=None, + schema=None): + from pandas_gbq import _load - for index, row in dataframe.reset_index(drop=True).iterrows(): - row_json = row.to_json( - force_ascii=False, date_unit='s', date_format='iso') - rows.append(row_json) - remaining_rows -= 1 + total_rows = len(dataframe) + self._print("\n\n") - if (len(rows) % chunksize == 0) or (remaining_rows == 0): + try: + for remaining_rows in _load.load_chunks( + self.client, dataframe, dataset_id, table_id, + chunksize=chunksize): self._print("\rLoad is {0}% Complete".format( ((total_rows - remaining_rows) * 100) / total_rows)) - - body = '{}\n'.format('\n'.join(rows)) - if isinstance(body, bytes): - body = body.decode('utf-8') - body = body.encode('utf-8') - body = BytesIO(body) - - try: - self.client.load_table_from_file( - body, - destination_table, - job_config=job_config).result() - except self.http_error as ex: - self.process_http_error(ex) - - rows = [] + except self.http_error as ex: + self.process_http_error(ex) self._print("\n") @@ -888,7 +865,7 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, return final_df -def to_gbq(dataframe, destination_table, project_id, chunksize=10000, +def to_gbq(dataframe, destination_table, project_id, chunksize=None, verbose=True, reauth=False, if_exists='fail', private_key=None, auth_local_webserver=False, table_schema=None): """Write a DataFrame to a Google BigQuery table. @@ -922,8 +899,9 @@ def to_gbq(dataframe, destination_table, project_id, chunksize=10000, Name of table to be written, in the form 'dataset.tablename' project_id : str Google BigQuery Account project ID. - chunksize : int (default 10000) - Number of rows to be inserted in each chunk from the dataframe. + chunksize : int (default None) + Number of rows to be inserted in each chunk from the dataframe. Use + ``None`` to load the dataframe in a single chunk. verbose : boolean (default True) Show percentage complete reauth : boolean (default False) @@ -985,7 +963,7 @@ def to_gbq(dataframe, destination_table, project_id, chunksize=10000, raise TableCreationError("Could not create the table because it " "already exists. " "Change the if_exists parameter to " - "append or replace data.") + "'append' or 'replace' data.") elif if_exists == 'replace': connector.delete_and_recreate_table( dataset_id, table_id, table_schema) @@ -999,19 +977,14 @@ def to_gbq(dataframe, destination_table, project_id, chunksize=10000, else: table.create(table_id, table_schema) - connector.load_data(dataframe, dataset_id, table_id, chunksize) + connector.load_data( + dataframe, dataset_id, table_id, chunksize=chunksize, + schema=table_schema) def generate_bq_schema(df, default_type='STRING'): - # deprecation TimeSeries, #11121 - warnings.warn("generate_bq_schema is deprecated and will be removed in " - "a future version", FutureWarning, stacklevel=2) - - return _generate_bq_schema(df, default_type=default_type) - - -def _generate_bq_schema(df, default_type='STRING'): - """ Given a passed df, generate the associated Google BigQuery schema. + """DEPRECATED: Given a passed df, generate the associated Google BigQuery + schema. Parameters ---------- @@ -1020,23 +993,16 @@ def _generate_bq_schema(df, default_type='STRING'): The default big query type in case the type of the column does not exist in the schema. """ + # deprecation TimeSeries, #11121 + warnings.warn("generate_bq_schema is deprecated and will be removed in " + "a future version", FutureWarning, stacklevel=2) - type_mapping = { - 'i': 'INTEGER', - 'b': 'BOOLEAN', - 'f': 'FLOAT', - 'O': 'STRING', - 'S': 'STRING', - 'U': 'STRING', - 'M': 'TIMESTAMP' - } + return _generate_bq_schema(df, default_type=default_type) - fields = [] - for column_name, dtype in df.dtypes.iteritems(): - fields.append({'name': column_name, - 'type': type_mapping.get(dtype.kind, default_type)}) - return {'fields': fields} +def _generate_bq_schema(df, default_type='STRING'): + from pandas_gbq import _schema + return _schema.generate_bq_schema(df, default_type=default_type) class _Table(GbqConnector): @@ -1096,6 +1062,9 @@ def create(self, table_id, schema): table_ref = self.client.dataset(self.dataset_id).table(table_id) table = Table(table_ref) + # Manually create the schema objects, adding NULLABLE mode + # as a workaround for + # https://github.com/GoogleCloudPlatform/google-cloud-python/issues/4456 for field in schema['fields']: if 'mode' not in field: field['mode'] = 'NULLABLE' diff --git a/packages/pandas-gbq/pandas_gbq/tests/test__load.py b/packages/pandas-gbq/pandas_gbq/tests/test__load.py new file mode 100644 index 000000000000..d63638e2fa39 --- /dev/null +++ b/packages/pandas-gbq/pandas_gbq/tests/test__load.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- + +import numpy +import pandas + + +def test_encode_chunk_with_unicode(): + """Test that a dataframe containing unicode can be encoded as a file. + + See: https://github.com/pydata/pandas-gbq/issues/106 + """ + from pandas_gbq._load import encode_chunk + + df = pandas.DataFrame( + numpy.random.randn(6, 4), index=range(6), columns=list('ABCD')) + df['s'] = u'信用卡' + csv_buffer = encode_chunk(df) + csv_bytes = csv_buffer.read() + csv_string = csv_bytes.decode('utf-8') + assert u'信用卡' in csv_string + + +def test_encode_chunks_splits_dataframe(): + from pandas_gbq._load import encode_chunks + df = pandas.DataFrame(numpy.random.randn(6, 4), index=range(6)) + chunks = list(encode_chunks(df, chunksize=2)) + assert len(chunks) == 3 + remaining, buffer = chunks[0] + assert remaining == 4 + assert len(buffer.readlines()) == 2 + + +def test_encode_chunks_with_chunksize_none(): + from pandas_gbq._load import encode_chunks + df = pandas.DataFrame(numpy.random.randn(6, 4), index=range(6)) + chunks = list(encode_chunks(df)) + assert len(chunks) == 1 + remaining, buffer = chunks[0] + assert remaining == 0 + assert len(buffer.readlines()) == 6 diff --git a/packages/pandas-gbq/pandas_gbq/tests/test__schema.py b/packages/pandas-gbq/pandas_gbq/tests/test__schema.py new file mode 100644 index 000000000000..5c7fffc1cbff --- /dev/null +++ b/packages/pandas-gbq/pandas_gbq/tests/test__schema.py @@ -0,0 +1,55 @@ + +import datetime + +import pandas +import pytest + +from pandas_gbq import _schema + + +@pytest.mark.parametrize( + 'dataframe,expected_schema', + [ + ( + pandas.DataFrame(data={'col1': [1, 2, 3]}), + {'fields': [{'name': 'col1', 'type': 'INTEGER'}]}, + ), + ( + pandas.DataFrame(data={'col1': [True, False]}), + {'fields': [{'name': 'col1', 'type': 'BOOLEAN'}]}, + ), + ( + pandas.DataFrame(data={'col1': [1.0, 3.14]}), + {'fields': [{'name': 'col1', 'type': 'FLOAT'}]}, + ), + ( + pandas.DataFrame(data={'col1': [u'hello', u'world']}), + {'fields': [{'name': 'col1', 'type': 'STRING'}]}, + ), + ( + pandas.DataFrame(data={'col1': [datetime.datetime.now()]}), + {'fields': [{'name': 'col1', 'type': 'TIMESTAMP'}]}, + ), + ( + pandas.DataFrame( + data={ + 'col1': [datetime.datetime.now()], + 'col2': [u'hello'], + 'col3': [3.14], + 'col4': [True], + 'col5': [4], + }), + { + 'fields': [ + {'name': 'col1', 'type': 'TIMESTAMP'}, + {'name': 'col2', 'type': 'STRING'}, + {'name': 'col3', 'type': 'FLOAT'}, + {'name': 'col4', 'type': 'BOOLEAN'}, + {'name': 'col5', 'type': 'INTEGER'}, + ], + }, + ), + ]) +def test_generate_bq_schema(dataframe, expected_schema): + schema = _schema.generate_bq_schema(dataframe) + assert schema == expected_schema diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py index f4f731b1144c..97f4729ec0c8 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -1218,6 +1218,29 @@ def test_upload_other_unicode_data(self): tm.assert_numpy_array_equal(expected.values, result.values) + def test_upload_mixed_float_and_int(self): + """Test that we can upload a dataframe containing an int64 and float64 column. + See: https://github.com/pydata/pandas-gbq/issues/116 + """ + test_id = "mixed_float_and_int" + test_size = 2 + df = DataFrame( + [[1, 1.1], [2, 2.2]], + index=['row 1', 'row 2'], + columns=['intColumn', 'floatColumn']) + + gbq.to_gbq( + df, self.destination_table + test_id, + _get_project_id(), + private_key=_get_private_key_path()) + + result_df = gbq.read_gbq("SELECT * FROM {0}".format( + self.destination_table + test_id), + project_id=_get_project_id(), + private_key=_get_private_key_path()) + + assert len(result_df) == test_size + def test_generate_schema(self): df = tm.makeMixedDataFrame() schema = gbq._generate_bq_schema(df) From 4c62fe818aca119a893e20e6dc2dbf2f4837b68c Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 13 Feb 2018 12:54:16 -0800 Subject: [PATCH 099/519] Release 0.3.1 --- packages/pandas-gbq/docs/source/changelog.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index c3d0aa743acd..4720cd936dfb 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,8 +1,7 @@ Changelog ========= - -0.3.1 / [TBD] +0.3.1 / 2018-02-13 ------------------ - Fix an issue where Unicode couldn't be uploaded in Python 2 (:issue:`93`) From bead816d5481ac0dd145357f1e056333064a8fc9 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 13 Feb 2018 13:15:54 -0800 Subject: [PATCH 100/519] Add instructions for using test PyPI --- packages/pandas-gbq/release-procedure.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/release-procedure.md b/packages/pandas-gbq/release-procedure.md index c0bc1ab22078..da4b011ae670 100644 --- a/packages/pandas-gbq/release-procedure.md +++ b/packages/pandas-gbq/release-procedure.md @@ -6,10 +6,20 @@ git push pandas-gbq master --tags +* Build the package + + twine upload dist/* + +* Upload to test PyPI + + twine upload --repository testpypi dist/* + +* Try out test PyPI package + + pip install --upgrade --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple pandas-gbq + * Upload to PyPI - git clean -xfd - python setup.py register sdist bdist_wheel --universal twine upload dist/* * Do a pull-request to the feedstock on `pandas-gbq-feedstock `__ From 41dbe0cd7c1b69ac39cfb6b15fb709f9225126a9 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 13 Feb 2018 13:24:20 -0800 Subject: [PATCH 101/519] DOC: unicode error references correct issue --- packages/pandas-gbq/docs/source/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 4720cd936dfb..ae38ca8cebc0 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -4,7 +4,7 @@ Changelog 0.3.1 / 2018-02-13 ------------------ -- Fix an issue where Unicode couldn't be uploaded in Python 2 (:issue:`93`) +- Fix an issue where Unicode couldn't be uploaded in Python 2 (:issue:`106`) - Add support for a passed schema in :func:``to_gbq`` instead inferring the schema from the passed ``DataFrame`` with ``DataFrame.dtypes`` (:issue:`46`) - Fix an issue where a dataframe containing both integer and floating point columns could not be uploaded with ``to_gbq`` (:issue:`116`) - ``to_gbq`` now uses ``to_csv`` to avoid manually looping over rows in a dataframe (should result in faster table uploads) (:issue:`96`) From 51948d8f0cf814a7d9c5766c894126f09c66ec1a Mon Sep 17 00:00:00 2001 From: Max Belov Date: Thu, 15 Feb 2018 20:52:05 +0300 Subject: [PATCH 102/519] Explicitly use 64bit integers (#121) On 64-bit windows 10 the `int` values are cast into 32-bit integers for reasons given here https://stackoverflow.com/questions/36278590/numpy-array-dtype-is-coming-as-int32-by-default-in-a-windows-10-64-bit-machine This leads to unhanded exceptions when handling values beyond 32-bit signed range In Bigquery, int is always 64bit, so the solution is to specify the data type explicitly --- packages/pandas-gbq/pandas_gbq/gbq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 8ccd2511dddf..382f276b45f8 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -848,7 +848,7 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, # cast BOOLEAN and INTEGER columns from object to bool/int # if they dont have any nulls AND field mode is not repeated (i.e., array) - type_map = {'BOOLEAN': bool, 'INTEGER': int} + type_map = {'BOOLEAN': bool, 'INTEGER': np.int64} for field in schema['fields']: if field['type'].upper() in type_map and \ final_df[field['name']].notnull().all() and \ From 5ba2d725bf7254a2ba1a0179fcfb153c85b93653 Mon Sep 17 00:00:00 2001 From: Matti Remes Date: Tue, 20 Feb 2018 14:05:30 +0200 Subject: [PATCH 103/519] Fix package description (#130) --- packages/pandas-gbq/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index 3aa4b1706c04..3823ea3893c1 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -29,7 +29,7 @@ def readme(): name=NAME, version=versioneer.get_version(), cmdclass=versioneer.get_cmdclass(), - description="Pandas interface to Google Big Query", + description="Pandas interface to Google BigQuery", long_description=readme(), license='BSD License', author='The PyData Development Team', From eceb9061d79448a62eaf9655e73cad0b839a13a0 Mon Sep 17 00:00:00 2001 From: "Jason Q. Ng" Date: Fri, 23 Feb 2018 05:59:13 -0500 Subject: [PATCH 104/519] BUG: Use object dtype when querying an array of floats (#134) * Fix array of floats bug * Update changelog --- packages/pandas-gbq/docs/source/changelog.rst | 4 ++++ packages/pandas-gbq/pandas_gbq/gbq.py | 2 ++ packages/pandas-gbq/pandas_gbq/tests/test_gbq.py | 8 ++++++++ 3 files changed, 14 insertions(+) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index ae38ca8cebc0..eca6f9ce664d 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,6 +1,10 @@ Changelog ========= +0.3.2 / [TBD] +------------------ +- Fix bug with querying for an array of floats (:issue:`123`) + 0.3.1 / 2018-02-13 ------------------ diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 382f276b45f8..c8eed83b1fd9 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -720,6 +720,8 @@ def _parse_data(schema, rows): col_names = [str(field['name']) for field in fields] col_dtypes = [ dtype_map.get(field['type'].upper(), object) + if field['mode'] != 'repeated' + else object for field in fields ] page_array = np.zeros((len(rows),), dtype=lzip(col_names, col_dtypes)) diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py index 97f4729ec0c8..e1bedef01482 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -973,6 +973,14 @@ def test_array_agg(self): tm.assert_frame_equal(df, DataFrame([["a", [1, 3]], ["b", [2]]], columns=["letter", "numbers"])) + def test_array_of_floats(self): + query = """select [1.1, 2.2, 3.3] as a, 4 as b""" + df = gbq.read_gbq(query, project_id=_get_project_id(), + private_key=_get_private_key_path(), + dialect='standard') + tm.assert_frame_equal(df, DataFrame([[[1.1, 2.2, 3.3], 4]], + columns=["a", "b"])) + class TestToGBQIntegrationWithServiceAccountKeyPath(object): # Changes to BigQuery table schema may take up to 2 minutes as of May 2015 From 829bebe4eedae31df86a0267b338220a27f7a0cc Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+maxim-lian@users.noreply.github.com> Date: Fri, 23 Feb 2018 06:07:41 -0500 Subject: [PATCH 105/519] fix test_parse test (#126) --- packages/pandas-gbq/pandas_gbq/tests/test_gbq.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py index e1bedef01482..aceb9d7c113a 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -365,12 +365,16 @@ def test_read_gbq_with_no_project_id_given_should_fail(self): gbq.read_gbq('SELECT 1') def test_that_parse_data_works_properly(self): + + from google.cloud.bigquery.table import Row test_schema = {'fields': [ - {'mode': 'NULLABLE', 'name': 'valid_string', 'type': 'STRING'}]} - test_page = [{'f': [{'v': 'PI'}]}] + {'mode': 'NULLABLE', 'name': 'column_x', 'type': 'STRING'}]} + field_to_index = {'column_x': 0} + values = ('row_value',) + test_page = [Row(values, field_to_index)] test_output = gbq._parse_data(test_schema, test_page) - correct_output = DataFrame({'valid_string': ['PI']}) + correct_output = DataFrame({'column_x': ['row_value']}) tm.assert_frame_equal(test_output, correct_output) def test_read_gbq_with_invalid_private_key_json_should_fail(self): From c5ab5c92adc9cc39186da05ac6f3562e630f76da Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+maxim-lian@users.noreply.github.com> Date: Sun, 4 Mar 2018 07:41:43 -0500 Subject: [PATCH 106/519] BLD: add @stickler-ci (#141) --- packages/pandas-gbq/.stickler.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 packages/pandas-gbq/.stickler.yml diff --git a/packages/pandas-gbq/.stickler.yml b/packages/pandas-gbq/.stickler.yml new file mode 100644 index 000000000000..1a31b479c0ca --- /dev/null +++ b/packages/pandas-gbq/.stickler.yml @@ -0,0 +1,6 @@ +linters: + flake8: + max-line-length: 79 + fixer: true +fixers: + enable: true From d85d9d7775f38d11c547ec6824dc993631788432 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+maxim-lian@users.noreply.github.com> Date: Wed, 7 Mar 2018 19:07:32 -0500 Subject: [PATCH 107/519] TST: use pytest idioms (#139) * remove _setup_common * gitignore additions * !fix _setup_common * lint * project as a fixture * lint * gitignore additions * change setup_method to autouse fixture * mock breaking on Py3 * remove some empty methods * Revert "gitignore additions" This reverts commit 5abd4ecd96fafee67d9bd8c9ce2db1e664389cad. * skip implicitly with no project id when required * integrate the integration tests * typo * connector fixture * param another test * All-but-eliminated auth-specific tests * put teardown code in the fixture * scope=function * and in travis * Only run one auth type in Travis --- packages/pandas-gbq/.gitignore | 4 +- .../pandas-gbq/ci/requirements-3.5-0.18.1.pip | 2 +- .../pandas-gbq/pandas_gbq/tests/test_gbq.py | 615 ++++++------------ 3 files changed, 198 insertions(+), 423 deletions(-) diff --git a/packages/pandas-gbq/.gitignore b/packages/pandas-gbq/.gitignore index deba4dd80388..aca37f640560 100644 --- a/packages/pandas-gbq/.gitignore +++ b/packages/pandas-gbq/.gitignore @@ -1,4 +1,4 @@ -gi######################################### +######################################### # Editor temporary/working/backup files # .#* *\#*\# @@ -19,6 +19,8 @@ gi######################################### .noseids .ipynb_checkpoints .tags +.pytest_cache +.testmondata # Docs # ######## diff --git a/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip b/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip index ec27d3cc89f3..f895fb1f1a62 100644 --- a/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip +++ b/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip @@ -1,4 +1,4 @@ -google-auth==1.0.0 +google-auth==1.0.2 google-auth-oauthlib==0.0.1 mock google-cloud-bigquery==0.29.0 diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py index aceb9d7c113a..49afdf5783ef 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -1,36 +1,29 @@ # -*- coding: utf-8 -*- -import pytest - +import os import re +import sys from datetime import datetime -import pytz -from time import sleep -import os from random import randint -import logging -import sys +from time import sleep import numpy as np - -from pandas import compat - -from pandas.compat import u, range -from pandas import NaT, DataFrame -from pandas_gbq import gbq import pandas.util.testing as tm +import pytest +import pytz +from pandas import DataFrame, NaT, compat +from pandas.compat import range, u from pandas.compat.numpy import np_datetime64_compat +from pandas_gbq import gbq +try: + import mock +except ImportError: + from unittest import mock TABLE_ID = 'new_test' -def _skip_if_no_project_id(): - if not _get_project_id(): - pytest.skip( - "Cannot run integration tests without a project id") - - def _skip_local_auth_if_in_travis_env(): if _in_travis_environment(): pytest.skip("Cannot run local auth in travis environment") @@ -58,7 +51,12 @@ def _get_dataset_prefix_random(): def _get_project_id(): - return os.environ.get('GBQ_PROJECT_ID') + + project = os.environ.get('GBQ_PROJECT_ID') + if not project: + pytest.skip( + "Cannot run integration tests without a project id") + return project def _get_private_key_path(): @@ -78,6 +76,7 @@ def _get_private_key_contents(): return f.read() +@pytest.fixture(autouse=True, scope='module') def _test_imports(): try: import pkg_resources # noqa @@ -87,15 +86,9 @@ def _test_imports(): gbq._test_google_api_imports() -def _setup_common(): - try: - _test_imports() - except (ImportError, NotImplementedError) as import_exception: - pytest.skip(str(import_exception)) - - if _in_travis_environment(): - logging.getLogger('oauth2client').setLevel(logging.ERROR) - logging.getLogger('apiclient').setLevel(logging.ERROR) +@pytest.fixture +def project(): + return _get_project_id() def _check_if_can_get_correct_default_credentials(): @@ -179,39 +172,84 @@ def test_generate_bq_schema_deprecated(): gbq.generate_bq_schema(df) -class TestGBQConnectorIntegrationWithLocalUserAccountAuth(object): +@pytest.fixture(params=['local', 'service_path', 'service_creds']) +def auth_type(request): + + auth = request.param + + if auth == 'local': + + if _in_travis_environment(): + pytest.skip("Cannot run local auth in travis environment") + + elif auth == 'service_path': + + if _in_travis_environment(): + pytest.skip("Only run one auth type in Travis to save time") + + _skip_if_no_private_key_path() + elif auth == 'service_creds': + _skip_if_no_private_key_contents() + else: + raise ValueError + return auth - def setup_method(self, method): - _setup_common() - _skip_if_no_project_id() - _skip_local_auth_if_in_travis_env() - self.sut = gbq.GbqConnector( - _get_project_id(), auth_local_webserver=True) +@pytest.fixture() +def credentials(auth_type): - def test_should_be_able_to_make_a_connector(self): - assert self.sut is not None, 'Could not create a GbqConnector' + if auth_type == 'local': + return None + + elif auth_type == 'service_path': + return _get_private_key_path() + elif auth_type == 'service_creds': + return _get_private_key_contents() + else: + raise ValueError + + +@pytest.fixture() +def gbq_connector(project, credentials): + + return gbq.GbqConnector(project, private_key=credentials) - def test_should_be_able_to_get_valid_credentials(self): - credentials = self.sut.get_credentials() + +class TestGBQConnectorIntegration(object): + + def test_should_be_able_to_make_a_connector(self, gbq_connector): + assert gbq_connector is not None, 'Could not create a GbqConnector' + + def test_should_be_able_to_get_valid_credentials(self, gbq_connector): + credentials = gbq_connector.get_credentials() assert credentials.valid - def test_should_be_able_to_get_a_bigquery_client(self): - bigquery_client = self.sut.get_client() + def test_should_be_able_to_get_a_bigquery_client(self, gbq_connector): + bigquery_client = gbq_connector.get_client() assert bigquery_client is not None - def test_should_be_able_to_get_schema_from_query(self): - schema, pages = self.sut.run_query('SELECT 1') + def test_should_be_able_to_get_schema_from_query(self, gbq_connector): + schema, pages = gbq_connector.run_query('SELECT 1') assert schema is not None - def test_should_be_able_to_get_results_from_query(self): - schema, pages = self.sut.run_query('SELECT 1') + def test_should_be_able_to_get_results_from_query(self, gbq_connector): + schema, pages = gbq_connector.run_query('SELECT 1') assert pages is not None + +class TestGBQConnectorIntegrationWithLocalUserAccountAuth(object): + + @pytest.fixture(autouse=True) + def setup(self, project): + + _skip_local_auth_if_in_travis_env() + + self.sut = gbq.GbqConnector(project, auth_local_webserver=True) + def test_get_application_default_credentials_does_not_throw_error(self): if _check_if_can_get_correct_default_credentials(): # Can get real credentials, so mock it out to fail. - import mock + from google.auth.exceptions import DefaultCredentialsError with mock.patch('google.auth.default', side_effect=DefaultCredentialsError()): @@ -229,7 +267,7 @@ def test_get_application_default_credentials_returns_credentials(self): assert isinstance(credentials, Credentials) def test_get_user_account_credentials_bad_file_returns_credentials(self): - import mock + from google.auth.credentials import Credentials with mock.patch('__main__.open', side_effect=IOError()): credentials = self.sut.get_user_account_credentials() @@ -241,73 +279,8 @@ def test_get_user_account_credentials_returns_credentials(self): assert isinstance(credentials, Credentials) -class TestGBQConnectorIntegrationWithServiceAccountKeyPath(object): - - def setup_method(self, method): - _setup_common() - - _skip_if_no_project_id() - _skip_if_no_private_key_path() - - self.sut = gbq.GbqConnector(_get_project_id(), - private_key=_get_private_key_path()) - - def test_should_be_able_to_make_a_connector(self): - assert self.sut is not None - - def test_should_be_able_to_get_valid_credentials(self): - credentials = self.sut.get_credentials() - assert credentials.valid - - def test_should_be_able_to_get_a_bigquery_client(self): - bigquery_client = self.sut.get_client() - assert bigquery_client is not None - - def test_should_be_able_to_get_schema_from_query(self): - schema, pages = self.sut.run_query('SELECT 1') - assert schema is not None - - def test_should_be_able_to_get_results_from_query(self): - schema, pages = self.sut.run_query('SELECT 1') - assert pages is not None - - -class TestGBQConnectorIntegrationWithServiceAccountKeyContents(object): - - def setup_method(self, method): - _setup_common() - - _skip_if_no_project_id() - _skip_if_no_private_key_contents() - - self.sut = gbq.GbqConnector(_get_project_id(), - private_key=_get_private_key_contents()) - - def test_should_be_able_to_make_a_connector(self): - assert self.sut is not None - - def test_should_be_able_to_get_valid_credentials(self): - credentials = self.sut.get_credentials() - assert credentials.valid - - def test_should_be_able_to_get_a_bigquery_client(self): - bigquery_client = self.sut.get_client() - assert bigquery_client is not None - - def test_should_be_able_to_get_schema_from_query(self): - schema, pages = self.sut.run_query('SELECT 1') - assert schema is not None - - def test_should_be_able_to_get_results_from_query(self): - schema, pages = self.sut.run_query('SELECT 1') - assert pages is not None - - class GBQUnitTests(object): - def setup_method(self, method): - _setup_common() - def test_import_google_api_python_client(self): if not _in_travis_environment(): pytest.skip("Skip if not in travis environment. Extra test to " @@ -325,7 +298,7 @@ def test_import_google_api_python_client(self): from googleapiclient.errors import HttpError # noqa def test_should_return_credentials_path_set_by_env_var(self): - import mock + env = {'PANDAS_GBQ_CREDENTIALS_FILE': '/tmp/dummy.dat'} with mock.patch.dict('os.environ', env): assert gbq._get_credentials_file() == '/tmp/dummy.dat' @@ -406,114 +379,46 @@ def test_read_gbq_with_corrupted_private_key_json_should_fail(self): private_key=re.sub('[a-z]', '9', _get_private_key_contents())) -class TestReadGBQIntegration(object): - - @classmethod - def setup_class(cls): - # - GLOBAL CLASS FIXTURES - - # put here any instruction you want to execute only *ONCE* *BEFORE* - # executing *ALL* tests described below. - - _skip_if_no_project_id() - - _setup_common() - - def setup_method(self, method): - # - PER-TEST FIXTURES - - # put here any instruction you want to be run *BEFORE* *EVERY* test is - # executed. - pass - - @classmethod - def teardown_class(cls): - # - GLOBAL CLASS FIXTURES - - # put here any instruction you want to execute only *ONCE* *AFTER* - # executing all tests. - pass - - def teardown_method(self, method): - # - PER-TEST FIXTURES - - # put here any instructions you want to be run *AFTER* *EVERY* test is - # executed. - pass - - def test_should_read_as_user_account(self): - _skip_local_auth_if_in_travis_env() - - query = 'SELECT "PI" AS valid_string' - df = gbq.read_gbq(query, project_id=_get_project_id()) - tm.assert_frame_equal(df, DataFrame({'valid_string': ['PI']})) - - def test_should_read_as_service_account_with_key_path(self): - _skip_if_no_private_key_path() - query = 'SELECT "PI" AS valid_string' - df = gbq.read_gbq(query, project_id=_get_project_id(), - private_key=_get_private_key_path()) - tm.assert_frame_equal(df, DataFrame({'valid_string': ['PI']})) - - def test_should_read_as_service_account_with_key_contents(self): - _skip_if_no_private_key_contents() - query = 'SELECT "PI" AS valid_string' - df = gbq.read_gbq(query, project_id=_get_project_id(), - private_key=_get_private_key_contents()) - tm.assert_frame_equal(df, DataFrame({'valid_string': ['PI']})) - - -class TestReadGBQIntegrationWithServiceAccountKeyPath(object): +def test_should_read(project, credentials): - @classmethod - def setup_class(cls): - # - GLOBAL CLASS FIXTURES - - # put here any instruction you want to execute only *ONCE* *BEFORE* - # executing *ALL* tests described below. + query = 'SELECT "PI" AS valid_string' + df = gbq.read_gbq(query, project_id=project, private_key=credentials) + tm.assert_frame_equal(df, DataFrame({'valid_string': ['PI']})) - _skip_if_no_project_id() - _skip_if_no_private_key_path() - _setup_common() +class TestReadGBQIntegration(object): - def setup_method(self, method): + @pytest.fixture(autouse=True) + def setup(self, project, credentials): # - PER-TEST FIXTURES - # put here any instruction you want to be run *BEFORE* *EVERY* test is # executed. self.gbq_connector = gbq.GbqConnector( - _get_project_id(), private_key=_get_private_key_path()) - - @classmethod - def teardown_class(cls): - # - GLOBAL CLASS FIXTURES - - # put here any instruction you want to execute only *ONCE* *AFTER* - # executing all tests. - pass - - def teardown_method(self): - # - PER-TEST FIXTURES - - # put here any instructions you want to be run *AFTER* *EVERY* test is - # executed. - pass + project, private_key=credentials) + self.credentials = credentials def test_should_properly_handle_valid_strings(self): query = 'SELECT "PI" AS valid_string' df = gbq.read_gbq(query, project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) tm.assert_frame_equal(df, DataFrame({'valid_string': ['PI']})) def test_should_properly_handle_empty_strings(self): query = 'SELECT "" AS empty_string' df = gbq.read_gbq(query, project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) tm.assert_frame_equal(df, DataFrame({'empty_string': [""]})) def test_should_properly_handle_null_strings(self): query = 'SELECT STRING(NULL) AS null_string' df = gbq.read_gbq(query, project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) tm.assert_frame_equal(df, DataFrame({'null_string': [None]})) def test_should_properly_handle_valid_integers(self): query = 'SELECT INTEGER(3) AS valid_integer' df = gbq.read_gbq(query, project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) tm.assert_frame_equal(df, DataFrame({'valid_integer': [3]})) def test_should_properly_handle_nullable_integers(self): @@ -521,14 +426,14 @@ def test_should_properly_handle_nullable_integers(self): (SELECT 1 AS nullable_integer), (SELECT NULL AS nullable_integer)''' df = gbq.read_gbq(query, project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) tm.assert_frame_equal( df, DataFrame({'nullable_integer': [1, None]}).astype(object)) def test_should_properly_handle_valid_longs(self): query = 'SELECT 1 << 62 AS valid_long' df = gbq.read_gbq(query, project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) tm.assert_frame_equal( df, DataFrame({'valid_long': [1 << 62]})) @@ -537,21 +442,21 @@ def test_should_properly_handle_nullable_longs(self): (SELECT 1 << 62 AS nullable_long), (SELECT NULL AS nullable_long)''' df = gbq.read_gbq(query, project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) tm.assert_frame_equal( df, DataFrame({'nullable_long': [1 << 62, None]}).astype(object)) def test_should_properly_handle_null_integers(self): query = 'SELECT INTEGER(NULL) AS null_integer' df = gbq.read_gbq(query, project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) tm.assert_frame_equal(df, DataFrame({'null_integer': [None]})) def test_should_properly_handle_valid_floats(self): from math import pi query = 'SELECT PI() AS valid_float' df = gbq.read_gbq(query, project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) tm.assert_frame_equal(df, DataFrame( {'valid_float': [pi]})) @@ -561,7 +466,7 @@ def test_should_properly_handle_nullable_floats(self): (SELECT PI() AS nullable_float), (SELECT NULL AS nullable_float)''' df = gbq.read_gbq(query, project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) tm.assert_frame_equal( df, DataFrame({'nullable_float': [pi, None]})) @@ -569,7 +474,7 @@ def test_should_properly_handle_valid_doubles(self): from math import pi query = 'SELECT PI() * POW(10, 307) AS valid_double' df = gbq.read_gbq(query, project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) tm.assert_frame_equal(df, DataFrame( {'valid_double': [pi * 10 ** 307]})) @@ -579,27 +484,27 @@ def test_should_properly_handle_nullable_doubles(self): (SELECT PI() * POW(10, 307) AS nullable_double), (SELECT NULL AS nullable_double)''' df = gbq.read_gbq(query, project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) tm.assert_frame_equal( df, DataFrame({'nullable_double': [pi * 10 ** 307, None]})) def test_should_properly_handle_null_floats(self): query = 'SELECT FLOAT(NULL) AS null_float' df = gbq.read_gbq(query, project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) tm.assert_frame_equal(df, DataFrame({'null_float': [np.nan]})) def test_should_properly_handle_timestamp_unix_epoch(self): query = 'SELECT TIMESTAMP("1970-01-01 00:00:00") AS unix_epoch' df = gbq.read_gbq(query, project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) tm.assert_frame_equal(df, DataFrame( {'unix_epoch': [np.datetime64('1970-01-01T00:00:00.000000Z')]})) def test_should_properly_handle_arbitrary_timestamp(self): query = 'SELECT TIMESTAMP("2004-09-15 05:00:00") AS valid_timestamp' df = gbq.read_gbq(query, project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) tm.assert_frame_equal(df, DataFrame({ 'valid_timestamp': [np.datetime64('2004-09-15T05:00:00.000000Z')] })) @@ -607,25 +512,25 @@ def test_should_properly_handle_arbitrary_timestamp(self): def test_should_properly_handle_null_timestamp(self): query = 'SELECT TIMESTAMP(NULL) AS null_timestamp' df = gbq.read_gbq(query, project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) tm.assert_frame_equal(df, DataFrame({'null_timestamp': [NaT]})) def test_should_properly_handle_true_boolean(self): query = 'SELECT BOOLEAN(TRUE) AS true_boolean' df = gbq.read_gbq(query, project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) tm.assert_frame_equal(df, DataFrame({'true_boolean': [True]})) def test_should_properly_handle_false_boolean(self): query = 'SELECT BOOLEAN(FALSE) AS false_boolean' df = gbq.read_gbq(query, project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) tm.assert_frame_equal(df, DataFrame({'false_boolean': [False]})) def test_should_properly_handle_null_boolean(self): query = 'SELECT BOOLEAN(NULL) AS null_boolean' df = gbq.read_gbq(query, project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) tm.assert_frame_equal(df, DataFrame({'null_boolean': [None]})) def test_should_properly_handle_nullable_booleans(self): @@ -633,7 +538,7 @@ def test_should_properly_handle_nullable_booleans(self): (SELECT BOOLEAN(TRUE) AS nullable_boolean), (SELECT NULL AS nullable_boolean)''' df = gbq.read_gbq(query, project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) tm.assert_frame_equal( df, DataFrame({'nullable_boolean': [True, None]}).astype(object)) @@ -650,14 +555,14 @@ def test_unicode_string_conversion_and_normalization(self): query = 'SELECT "{0}" AS unicode_string'.format(unicode_string) df = gbq.read_gbq(query, project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) tm.assert_frame_equal(df, correct_test_datatype) def test_index_column(self): query = "SELECT 'a' AS string_1, 'b' AS string_2" result_frame = gbq.read_gbq(query, project_id=_get_project_id(), index_col="string_1", - private_key=_get_private_key_path()) + private_key=self.credentials) correct_frame = DataFrame( {'string_1': ['a'], 'string_2': ['b']}).set_index("string_1") assert result_frame.index.name == correct_frame.index.name @@ -667,7 +572,7 @@ def test_column_order(self): col_order = ['string_3', 'string_1', 'string_2'] result_frame = gbq.read_gbq(query, project_id=_get_project_id(), col_order=col_order, - private_key=_get_private_key_path()) + private_key=self.credentials) correct_frame = DataFrame({'string_1': ['a'], 'string_2': [ 'b'], 'string_3': ['c']})[col_order] tm.assert_frame_equal(result_frame, correct_frame) @@ -680,14 +585,14 @@ def test_read_gbq_raises_invalid_column_order(self): with pytest.raises(gbq.InvalidColumnOrder): gbq.read_gbq(query, project_id=_get_project_id(), col_order=col_order, - private_key=_get_private_key_path()) + private_key=self.credentials) def test_column_order_plus_index(self): query = "SELECT 'a' AS string_1, 'b' AS string_2, 'c' AS string_3" col_order = ['string_3', 'string_2'] result_frame = gbq.read_gbq(query, project_id=_get_project_id(), index_col='string_1', col_order=col_order, - private_key=_get_private_key_path()) + private_key=self.credentials) correct_frame = DataFrame( {'string_1': ['a'], 'string_2': ['b'], 'string_3': ['c']}) correct_frame.set_index('string_1', inplace=True) @@ -702,24 +607,24 @@ def test_read_gbq_raises_invalid_index_column(self): with pytest.raises(gbq.InvalidIndexColumn): gbq.read_gbq(query, project_id=_get_project_id(), index_col='string_bbb', col_order=col_order, - private_key=_get_private_key_path()) + private_key=self.credentials) def test_malformed_query(self): with pytest.raises(gbq.GenericGBQException): gbq.read_gbq("SELCET * FORM [publicdata:samples.shakespeare]", project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) def test_bad_project_id(self): with pytest.raises(gbq.GenericGBQException): gbq.read_gbq("SELECT 1", project_id='001', - private_key=_get_private_key_path()) + private_key=self.credentials) def test_bad_table_name(self): with pytest.raises(gbq.GenericGBQException): gbq.read_gbq("SELECT * FROM [publicdata:samples.nope]", project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) def test_download_dataset_larger_than_200k_rows(self): test_size = 200005 @@ -729,7 +634,7 @@ def test_download_dataset_larger_than_200k_rows(self): "GROUP EACH BY id ORDER BY id ASC LIMIT {0}" .format(test_size), project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) assert len(df.drop_duplicates()) == test_size def test_zero_rows(self): @@ -739,7 +644,7 @@ def test_zero_rows(self): "FROM [publicdata:samples.wikipedia] " "WHERE timestamp=-9999999", project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) page_array = np.zeros( (0,), dtype=[('title', object), ('id', np.dtype(int)), ('is_bot', np.dtype(bool)), ('ts', 'M8[ns]')]) @@ -755,13 +660,13 @@ def test_legacy_sql(self): with pytest.raises(gbq.GenericGBQException): gbq.read_gbq(legacy_sql, project_id=_get_project_id(), dialect='standard', - private_key=_get_private_key_path()) + private_key=self.credentials) # Test that a legacy sql statement succeeds when # setting dialect='legacy' df = gbq.read_gbq(legacy_sql, project_id=_get_project_id(), dialect='legacy', - private_key=_get_private_key_path()) + private_key=self.credentials) assert len(df.drop_duplicates()) == 10 def test_standard_sql(self): @@ -772,13 +677,13 @@ def test_standard_sql(self): # the legacy SQL dialect (default value) with pytest.raises(gbq.GenericGBQException): gbq.read_gbq(standard_sql, project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) # Test that a standard sql statement succeeds when # setting dialect='standard' df = gbq.read_gbq(standard_sql, project_id=_get_project_id(), dialect='standard', - private_key=_get_private_key_path()) + private_key=self.credentials) assert len(df.drop_duplicates()) == 10 def test_invalid_option_for_sql_dialect(self): @@ -789,12 +694,12 @@ def test_invalid_option_for_sql_dialect(self): with pytest.raises(ValueError): gbq.read_gbq(sql_statement, project_id=_get_project_id(), dialect='invalid', - private_key=_get_private_key_path()) + private_key=self.credentials) # Test that a correct option for dialect succeeds # to make sure ValueError was due to invalid dialect gbq.read_gbq(sql_statement, project_id=_get_project_id(), - dialect='standard', private_key=_get_private_key_path()) + dialect='standard', private_key=self.credentials) def test_query_with_parameters(self): sql_statement = "SELECT @param1 + @param2 AS valid_result" @@ -828,12 +733,12 @@ def test_query_with_parameters(self): # when parameters are not supplied via configuration with pytest.raises(ValueError): gbq.read_gbq(sql_statement, project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) # Test that the query is successful because we have supplied # the correct query parameters via the 'config' option df = gbq.read_gbq(sql_statement, project_id=_get_project_id(), - private_key=_get_private_key_path(), + private_key=self.credentials, configuration=config) tm.assert_frame_equal(df, DataFrame({'valid_result': [3]})) @@ -850,11 +755,11 @@ def test_query_inside_configuration(self): # inside config and as parameter with pytest.raises(ValueError): gbq.read_gbq(query_no_use, project_id=_get_project_id(), - private_key=_get_private_key_path(), + private_key=self.credentials, configuration=config) df = gbq.read_gbq(None, project_id=_get_project_id(), - private_key=_get_private_key_path(), + private_key=self.credentials, configuration=config) tm.assert_frame_equal(df, DataFrame({'valid_string': ['PI']})) @@ -878,7 +783,7 @@ def test_configuration_without_query(self): # nor 'copy','load','extract' with pytest.raises(ValueError): gbq.read_gbq(sql_statement, project_id=_get_project_id(), - private_key=_get_private_key_path(), + private_key=self.credentials, configuration=config) def test_configuration_raises_value_error_with_multiple_config(self): @@ -896,7 +801,7 @@ def test_configuration_raises_value_error_with_multiple_config(self): # Test that only ValueError is raised with multiple configurations with pytest.raises(ValueError): gbq.read_gbq(sql_statement, project_id=_get_project_id(), - private_key=_get_private_key_path(), + private_key=self.credentials, configuration=config) def test_timeout_configuration(self): @@ -909,7 +814,7 @@ def test_timeout_configuration(self): # Test that QueryTimeout error raises with pytest.raises(gbq.QueryTimeout): gbq.read_gbq(sql_statement, project_id=_get_project_id(), - private_key=_get_private_key_path(), + private_key=self.credentials, configuration=config) def test_query_response_bytes(self): @@ -931,18 +836,19 @@ def test_struct(self): query = """SELECT 1 int_field, STRUCT("a" as letter, 1 as num) struct_field""" df = gbq.read_gbq(query, project_id=_get_project_id(), - private_key=_get_private_key_path(), + private_key=self.credentials, dialect='standard') - tm.assert_frame_equal(df, DataFrame([[1, {"letter": "a", "num": 1}]], - columns=["int_field", "struct_field"])) + expected = DataFrame([[1, {"letter": "a", "num": 1}]], + columns=["int_field", "struct_field"]) + tm.assert_frame_equal(df, expected) def test_array(self): query = """select ["a","x","b","y","c","z"] as letters""" df = gbq.read_gbq(query, project_id=_get_project_id(), - private_key=_get_private_key_path(), + private_key=self.credentials, dialect='standard') tm.assert_frame_equal(df, DataFrame([[["a", "x", "b", "y", "c", "z"]]], - columns=["letters"])) + columns=["letters"])) def test_array_length_zero(self): query = """WITH t as ( @@ -954,10 +860,11 @@ def test_array_length_zero(self): from t order by letter ASC""" df = gbq.read_gbq(query, project_id=_get_project_id(), - private_key=_get_private_key_path(), + private_key=self.credentials, dialect='standard') - tm.assert_frame_equal(df, DataFrame([["a", [""], 1], ["b", [], 0]], - columns=["letter", "array_field", "len"])) + expected = DataFrame([["a", [""], 1], ["b", [], 0]], + columns=["letter", "array_field", "len"]) + tm.assert_frame_equal(df, expected) def test_array_agg(self): query = """WITH t as ( @@ -972,10 +879,10 @@ def test_array_agg(self): group by letter order by letter ASC""" df = gbq.read_gbq(query, project_id=_get_project_id(), - private_key=_get_private_key_path(), + private_key=self.credentials, dialect='standard') tm.assert_frame_equal(df, DataFrame([["a", [1, 3]], ["b", [2]]], - columns=["letter", "numbers"])) + columns=["letter", "numbers"])) def test_array_of_floats(self): query = """select [1.1, 2.2, 3.3] as a, 4 as b""" @@ -986,53 +893,33 @@ def test_array_of_floats(self): columns=["a", "b"])) -class TestToGBQIntegrationWithServiceAccountKeyPath(object): +class TestToGBQIntegration(object): # Changes to BigQuery table schema may take up to 2 minutes as of May 2015 # As a workaround to this issue, each test should use a unique table name. - # Make sure to modify the for loop range in the teardown_class when a new + # Make sure to modify the for loop range in the autouse fixture when a new # test is added See `Issue 191 # `__ - @classmethod - def setup_class(cls): - # - GLOBAL CLASS FIXTURES - - # put here any instruction you want to execute only *ONCE* *BEFORE* - # executing *ALL* tests described below. - - _skip_if_no_project_id() - _skip_if_no_private_key_path() - - _setup_common() - - def setup_method(self, method): + @pytest.fixture(autouse=True, scope='function') + def setup(self, project, credentials): # - PER-TEST FIXTURES - # put here any instruction you want to be run *BEFORE* *EVERY* test is # executed. self.dataset_prefix = _get_dataset_prefix_random() - clean_gbq_environment(self.dataset_prefix, _get_private_key_path()) - self.dataset = gbq._Dataset(_get_project_id(), - private_key=_get_private_key_path()) - self.table = gbq._Table(_get_project_id(), self.dataset_prefix + "1", - private_key=_get_private_key_path()) - self.sut = gbq.GbqConnector(_get_project_id(), - private_key=_get_private_key_path()) + clean_gbq_environment(self.dataset_prefix, credentials) + self.dataset = gbq._Dataset(project, + private_key=credentials) + self.table = gbq._Table(project, self.dataset_prefix + "1", + private_key=credentials) + self.sut = gbq.GbqConnector(project, + private_key=credentials) self.destination_table = "{0}{1}.{2}".format(self.dataset_prefix, "1", TABLE_ID) self.dataset.create(self.dataset_prefix + "1") - - @classmethod - def teardown_class(cls): - # - GLOBAL CLASS FIXTURES - - # put here any instruction you want to execute only *ONCE* *AFTER* - # executing all tests. - pass - - def teardown_method(self, method): - # - PER-TEST FIXTURES - - # put here any instructions you want to be run *AFTER* *EVERY* test is - # executed. - clean_gbq_environment(self.dataset_prefix, _get_private_key_path()) + self.credentials = credentials + yield + clean_gbq_environment(self.dataset_prefix, self.credentials) def test_upload_data(self): test_id = "1" @@ -1040,12 +927,12 @@ def test_upload_data(self): df = make_mixed_dataframe_v2(test_size) gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), - chunksize=10000, private_key=_get_private_key_path()) + chunksize=10000, private_key=self.credentials) result = gbq.read_gbq("SELECT COUNT(*) AS num_rows FROM {0}" .format(self.destination_table + test_id), project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) assert result['num_rows'][0] == test_size def test_upload_data_if_table_exists_fail(self): @@ -1057,12 +944,12 @@ def test_upload_data_if_table_exists_fail(self): # Test the default value of if_exists is 'fail' with pytest.raises(gbq.TableCreationError): gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) # Test the if_exists parameter with value 'fail' with pytest.raises(gbq.TableCreationError): gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), - if_exists='fail', private_key=_get_private_key_path()) + if_exists='fail', private_key=self.credentials) def test_upload_data_if_table_exists_append(self): test_id = "3" @@ -1072,23 +959,23 @@ def test_upload_data_if_table_exists_append(self): # Initialize table with sample data gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), - chunksize=10000, private_key=_get_private_key_path()) + chunksize=10000, private_key=self.credentials) # Test the if_exists parameter with value 'append' gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), - if_exists='append', private_key=_get_private_key_path()) + if_exists='append', private_key=self.credentials) result = gbq.read_gbq("SELECT COUNT(*) AS num_rows FROM {0}" .format(self.destination_table + test_id), project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) assert result['num_rows'][0] == test_size * 2 # Try inserting with a different schema, confirm failure with pytest.raises(gbq.InvalidSchema): gbq.to_gbq(df_different_schema, self.destination_table + test_id, _get_project_id(), if_exists='append', - private_key=_get_private_key_path()) + private_key=self.credentials) def test_upload_subset_columns_if_table_exists_append(self): # Issue 24: Upload is succesful if dataframe has columns @@ -1100,17 +987,17 @@ def test_upload_subset_columns_if_table_exists_append(self): # Initialize table with sample data gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), - chunksize=10000, private_key=_get_private_key_path()) + chunksize=10000, private_key=self.credentials) # Test the if_exists parameter with value 'append' gbq.to_gbq(df_subset_cols, self.destination_table + test_id, _get_project_id(), - if_exists='append', private_key=_get_private_key_path()) + if_exists='append', private_key=self.credentials) result = gbq.read_gbq("SELECT COUNT(*) AS num_rows FROM {0}" .format(self.destination_table + test_id), project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) assert result['num_rows'][0] == test_size * 2 # This test is currently failing intermittently due to changes in the @@ -1132,17 +1019,17 @@ def test_upload_data_if_table_exists_replace(self): # Initialize table with sample data gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), - chunksize=10000, private_key=_get_private_key_path()) + chunksize=10000, private_key=self.credentials) # Test the if_exists parameter with the value 'replace'. gbq.to_gbq(df_different_schema, self.destination_table + test_id, _get_project_id(), if_exists='replace', - private_key=_get_private_key_path()) + private_key=self.credentials) result = gbq.read_gbq("SELECT COUNT(*) AS num_rows FROM {0}" .format(self.destination_table + test_id), project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) assert result['num_rows'][0] == 5 def test_upload_data_if_table_exists_raises_value_error(self): @@ -1153,7 +1040,7 @@ def test_upload_data_if_table_exists_raises_value_error(self): # Test invalid value for if_exists parameter raises value error with pytest.raises(ValueError): gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), - if_exists='xxxxx', private_key=_get_private_key_path()) + if_exists='xxxxx', private_key=self.credentials) def test_google_upload_errors_should_raise_exception(self): raise pytest.skip("buggy test") @@ -1167,7 +1054,7 @@ def test_google_upload_errors_should_raise_exception(self): with pytest.raises(gbq.StreamingInsertError): gbq.to_gbq(bad_df, self.destination_table + test_id, - _get_project_id(), private_key=_get_private_key_path()) + _get_project_id(), private_key=self.credentials) def test_upload_chinese_unicode_data(self): test_id = "2" @@ -1179,13 +1066,13 @@ def test_upload_chinese_unicode_data(self): gbq.to_gbq( df, self.destination_table + test_id, _get_project_id(), - private_key=_get_private_key_path(), + private_key=self.credentials, chunksize=10000) result_df = gbq.read_gbq( "SELECT * FROM {0}".format(self.destination_table + test_id), project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) assert len(result_df) == test_size @@ -1212,13 +1099,13 @@ def test_upload_other_unicode_data(self): gbq.to_gbq( df, self.destination_table + test_id, _get_project_id(), - private_key=_get_private_key_path(), + private_key=self.credentials, chunksize=10000) result_df = gbq.read_gbq("SELECT * FROM {0}".format( self.destination_table + test_id), project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) assert len(result_df) == test_size @@ -1244,12 +1131,12 @@ def test_upload_mixed_float_and_int(self): gbq.to_gbq( df, self.destination_table + test_id, _get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) result_df = gbq.read_gbq("SELECT * FROM {0}".format( self.destination_table + test_id), project_id=_get_project_id(), - private_key=_get_private_key_path()) + private_key=self.credentials) assert len(result_df) == test_size @@ -1358,13 +1245,13 @@ def test_upload_data_flexible_column_order(self): # Initialize table with sample data gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), - chunksize=10000, private_key=_get_private_key_path()) + chunksize=10000, private_key=self.credentials) df_columns_reversed = df[df.columns[::-1]] gbq.to_gbq(df_columns_reversed, self.destination_table + test_id, _get_project_id(), if_exists='append', - private_key=_get_private_key_path()) + private_key=self.credentials) def test_verify_schema_ignores_field_mode(self): test_id = "14" @@ -1468,7 +1355,7 @@ def test_upload_data_with_valid_user_schema(self): {'name': 'D', 'type': 'TIMESTAMP'}] destination_table = self.destination_table + test_id gbq.to_gbq(df, destination_table, _get_project_id(), - private_key=_get_private_key_path(), + private_key=self.credentials, table_schema=test_schema) dataset, table = destination_table.split('.') assert self.table.verify_schema(dataset, table, @@ -1484,7 +1371,7 @@ def test_upload_data_with_invalid_user_schema_raises_error(self): destination_table = self.destination_table + test_id with pytest.raises(gbq.GenericGBQException): gbq.to_gbq(df, destination_table, _get_project_id(), - private_key=_get_private_key_path(), + private_key=self.credentials, table_schema=test_schema) def test_upload_data_with_missing_schema_fields_raises_error(self): @@ -1496,7 +1383,7 @@ def test_upload_data_with_missing_schema_fields_raises_error(self): destination_table = self.destination_table + test_id with pytest.raises(gbq.GenericGBQException): gbq.to_gbq(df, destination_table, _get_project_id(), - private_key=_get_private_key_path(), + private_key=self.credentials, table_schema=test_schema) def test_list_dataset(self): @@ -1507,7 +1394,7 @@ def test_list_table_zero_results(self): dataset_id = self.dataset_prefix + "2" self.dataset.create(dataset_id) table_list = gbq._Dataset(_get_project_id(), - private_key=_get_private_key_path() + private_key=self.credentials ).tables(dataset_id) assert len(table_list) == 0 @@ -1549,117 +1436,3 @@ def create_table_data_dataset_does_not_exist(self): def test_dataset_does_not_exist(self): assert not self.dataset.exists(self.dataset_prefix + "_not_found") - - -class TestToGBQIntegrationWithLocalUserAccountAuth(object): - # Changes to BigQuery table schema may take up to 2 minutes as of May 2015 - # As a workaround to this issue, each test should use a unique table name. - # Make sure to modify the for loop range in the teardown_class when a new - # test is added - # See `Issue 191 - # `__ - - @classmethod - def setup_class(cls): - # - GLOBAL CLASS FIXTURES - - # put here any instruction you want to execute only *ONCE* *BEFORE* - # executing *ALL* tests described below. - - _skip_if_no_project_id() - _skip_local_auth_if_in_travis_env() - - _setup_common() - - def setup_method(self, method): - # - PER-TEST FIXTURES - - # put here any instruction you want to be run *BEFORE* *EVERY* test - # is executed. - - gbq.GbqConnector(_get_project_id(), auth_local_webserver=True) - self.dataset_prefix = _get_dataset_prefix_random() - clean_gbq_environment(self.dataset_prefix) - self.destination_table = "{0}{1}.{2}".format(self.dataset_prefix, "2", - TABLE_ID) - - @classmethod - def teardown_class(cls): - # - GLOBAL CLASS FIXTURES - - # put here any instruction you want to execute only *ONCE* *AFTER* - # executing all tests. - pass - - def teardown_method(self): - # - PER-TEST FIXTURES - - # put here any instructions you want to be run *AFTER* *EVERY* test - # is executed. - clean_gbq_environment(self.dataset_prefix, _get_private_key_path()) - - def test_upload_data(self): - test_id = "1" - test_size = 10 - df = make_mixed_dataframe_v2(test_size) - - gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), - chunksize=10000) - - result = gbq.read_gbq("SELECT COUNT(*) AS num_rows FROM {0}".format( - self.destination_table + test_id), - project_id=_get_project_id()) - - assert result['num_rows'][0] == test_size - - -class TestToGBQIntegrationWithServiceAccountKeyContents(object): - # Changes to BigQuery table schema may take up to 2 minutes as of May 2015 - # As a workaround to this issue, each test should use a unique table name. - # Make sure to modify the for loop range in the teardown_class when a new - # test is added - # See `Issue 191 - # `__ - - @classmethod - def setup_class(cls): - # - GLOBAL CLASS FIXTURES - - # put here any instruction you want to execute only *ONCE* *BEFORE* - # executing *ALL* tests described below. - - _setup_common() - _skip_if_no_project_id() - - _skip_if_no_private_key_contents() - - def setup_method(self, method): - # - PER-TEST FIXTURES - - # put here any instruction you want to be run *BEFORE* *EVERY* test - # is executed. - self.dataset_prefix = _get_dataset_prefix_random() - clean_gbq_environment(self.dataset_prefix, _get_private_key_contents()) - self.destination_table = "{0}{1}.{2}".format(self.dataset_prefix, "3", - TABLE_ID) - - @classmethod - def teardown_class(cls): - # - GLOBAL CLASS FIXTURES - - # put here any instruction you want to execute only *ONCE* *AFTER* - # executing all tests. - pass - - def teardown_method(self, method): - # - PER-TEST FIXTURES - - # put here any instructions you want to be run *AFTER* *EVERY* test - # is executed. - clean_gbq_environment(self.dataset_prefix, _get_private_key_contents()) - - def test_upload_data(self): - test_id = "1" - test_size = 10 - df = make_mixed_dataframe_v2(test_size) - - gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), - chunksize=10000, private_key=_get_private_key_contents()) - - result = gbq.read_gbq("SELECT COUNT(*) as num_rows FROM {0}".format( - self.destination_table + test_id), - project_id=_get_project_id(), - private_key=_get_private_key_contents()) - assert result['num_rows'][0] == test_size From 66e4eb46f37462341dc0f06eaec565ca408133ed Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 7 Mar 2018 19:08:50 -0500 Subject: [PATCH 108/519] CLN: Minor tweak to error message shown when dependencies are missing (#145) --- packages/pandas-gbq/pandas_gbq/gbq.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index c8eed83b1fd9..b918940bae1b 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -29,8 +29,7 @@ def _check_google_client_version(): if (StrictVersion(_BIGQUERY_CLIENT_VERSION) < StrictVersion(bigquery_client_minimum_version)): - raise ImportError('pandas requires google-cloud-bigquery >= {0} ' - 'for Google BigQuery support, ' + raise ImportError('pandas-gbq requires google-cloud-bigquery >= {0}, ' 'current version {1}' .format(bigquery_client_minimum_version, _BIGQUERY_CLIENT_VERSION)) @@ -42,22 +41,19 @@ def _test_google_api_imports(): from google_auth_oauthlib.flow import InstalledAppFlow # noqa except ImportError as ex: raise ImportError( - 'pandas requires google-auth-oauthlib for Google BigQuery ' - 'support: {0}'.format(ex)) + 'pandas-gbq requires google-auth-oauthlib: {0}'.format(ex)) try: import google.auth # noqa except ImportError as ex: raise ImportError( - "pandas requires google-auth for Google BigQuery support: " - "{0}".format(ex)) + "pandas-gbq requires google-auth: {0}".format(ex)) try: from google.cloud import bigquery # noqa except ImportError as ex: raise ImportError( - "pandas requires google-cloud-python for Google BigQuery support: " - "{0}".format(ex)) + "pandas-gbq requires google-cloud-bigquery: {0}".format(ex)) _check_google_client_version() From 92caa1c5c9a6cc9495f42627d5976884fea460e8 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+maxim-lian@users.noreply.github.com> Date: Thu, 8 Mar 2018 11:19:15 -0500 Subject: [PATCH 109/519] CLN: isort (#137) * sorted imports * gitignore additions * merge * flake --- packages/pandas-gbq/docs/source/conf.py | 1 + packages/pandas-gbq/pandas_gbq/_load.py | 2 +- packages/pandas-gbq/pandas_gbq/gbq.py | 13 ++++++------- packages/pandas-gbq/pandas_gbq/tests/test_gbq.py | 1 + packages/pandas-gbq/scripts/merge-py.py | 9 ++++----- packages/pandas-gbq/setup.cfg | 5 +++++ packages/pandas-gbq/setup.py | 3 +-- packages/pandas-gbq/versioneer.py | 10 ++++++---- 8 files changed, 25 insertions(+), 19 deletions(-) diff --git a/packages/pandas-gbq/docs/source/conf.py b/packages/pandas-gbq/docs/source/conf.py index 71cff5254d98..3dfd9f86e94c 100644 --- a/packages/pandas-gbq/docs/source/conf.py +++ b/packages/pandas-gbq/docs/source/conf.py @@ -18,6 +18,7 @@ # import os import sys + # sys.path.insert(0, os.path.abspath('.')) # -- General configuration ------------------------------------------------ diff --git a/packages/pandas-gbq/pandas_gbq/_load.py b/packages/pandas-gbq/pandas_gbq/_load.py index 45dfec586242..ac816cb15d4a 100644 --- a/packages/pandas-gbq/pandas_gbq/_load.py +++ b/packages/pandas-gbq/pandas_gbq/_load.py @@ -1,7 +1,7 @@ """Helper methods for loading data into BigQuery""" -from google.cloud import bigquery import six +from google.cloud import bigquery from pandas_gbq import _schema diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index b918940bae1b..606f2810ca31 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -1,15 +1,14 @@ -import warnings -from datetime import datetime import json +import os +import sys import time +import warnings +from datetime import datetime +from distutils.version import StrictVersion from time import sleep -import sys -import os import numpy as np - -from distutils.version import StrictVersion -from pandas import compat, DataFrame +from pandas import DataFrame, compat from pandas.compat import lzip diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py index 49afdf5783ef..2ddbea189674 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -14,6 +14,7 @@ from pandas import DataFrame, NaT, compat from pandas.compat import range, u from pandas.compat.numpy import np_datetime64_compat + from pandas_gbq import gbq try: diff --git a/packages/pandas-gbq/scripts/merge-py.py b/packages/pandas-gbq/scripts/merge-py.py index 16e5f822ab40..062ca6880f0c 100755 --- a/packages/pandas-gbq/scripts/merge-py.py +++ b/packages/pandas-gbq/scripts/merge-py.py @@ -25,15 +25,14 @@ from __future__ import print_function -from subprocess import check_output -from requests.auth import HTTPBasicAuth -import requests - import os -import six import sys import textwrap +from subprocess import check_output +import requests +import six +from requests.auth import HTTPBasicAuth from six.moves import input PANDASGBQ_HOME = '.' diff --git a/packages/pandas-gbq/setup.cfg b/packages/pandas-gbq/setup.cfg index 18c520a8669f..32e405568362 100644 --- a/packages/pandas-gbq/setup.cfg +++ b/packages/pandas-gbq/setup.cfg @@ -13,3 +13,8 @@ parentdir_prefix = pandas_gbq- [flake8] ignore = E731 + +[isort] +default_section=THIRDPARTY +known_first_party=pandas_gbq +multi_line_output=4 diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index 3823ea3893c1..a240cf45fa56 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -1,9 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from setuptools import setup, find_packages import versioneer - +from setuptools import find_packages, setup NAME = 'pandas-gbq' diff --git a/packages/pandas-gbq/versioneer.py b/packages/pandas-gbq/versioneer.py index 64fea1c89272..dffd66b69a6d 100644 --- a/packages/pandas-gbq/versioneer.py +++ b/packages/pandas-gbq/versioneer.py @@ -277,10 +277,7 @@ """ from __future__ import print_function -try: - import configparser -except ImportError: - import ConfigParser as configparser + import errno import json import os @@ -288,6 +285,11 @@ import subprocess import sys +try: + import configparser +except ImportError: + import ConfigParser as configparser + class VersioneerConfig: """Container for Versioneer configuration parameters.""" From 1402d00a1e591007dc8d033a03448c024f7eccf1 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+maxim-lian@users.noreply.github.com> Date: Fri, 9 Mar 2018 12:28:29 -0500 Subject: [PATCH 110/519] Skip failing test (#138) * Test incorrectly named * gitignore * skip failing test for the moment --- packages/pandas-gbq/.gitignore | 1 + packages/pandas-gbq/pandas_gbq/tests/test_gbq.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/.gitignore b/packages/pandas-gbq/.gitignore index aca37f640560..104e28dfa1aa 100644 --- a/packages/pandas-gbq/.gitignore +++ b/packages/pandas-gbq/.gitignore @@ -31,6 +31,7 @@ docs/source/_build .coverage coverage.xml coverage_html_report +.pytest_cache # Compiled source # ################### diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py index 2ddbea189674..eb2522536459 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -280,7 +280,9 @@ def test_get_user_account_credentials_returns_credentials(self): assert isinstance(credentials, Credentials) -class GBQUnitTests(object): +@pytest.mark.skip('Currently fails, see ' + 'https://github.com/pydata/pandas-gbq/pull/125') +class TestGBQUnit(object): def test_import_google_api_python_client(self): if not _in_travis_environment(): From 7c8aaeae7201e280760b2bd212f8bb6427d46232 Mon Sep 17 00:00:00 2001 From: rhoboro Date: Wed, 14 Mar 2018 04:40:35 +0900 Subject: [PATCH 111/519] BUG: TIMESTAMP fields were missing seconds and microseconds after `to_gbq`. (#148) * BUG: fix bug that all TIMESTAMP fileds become 00 secconds. * add a test for extact timestamp * use fixed value instead of now() --- packages/pandas-gbq/pandas_gbq/_load.py | 2 +- .../pandas-gbq/pandas_gbq/tests/test_gbq.py | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/pandas_gbq/_load.py b/packages/pandas-gbq/pandas_gbq/_load.py index ac816cb15d4a..c04bf7a6a8d6 100644 --- a/packages/pandas-gbq/pandas_gbq/_load.py +++ b/packages/pandas-gbq/pandas_gbq/_load.py @@ -15,7 +15,7 @@ def encode_chunk(dataframe): csv_buffer = six.StringIO() dataframe.to_csv( csv_buffer, index=False, header=False, encoding='utf-8', - date_format='%Y-%m-%d %H:%M') + date_format='%Y-%m-%d %H:%M:%S.%f') # Convert to a BytesIO buffer so that unicode text is properly handled. # See: https://github.com/pydata/pandas-gbq/issues/106 diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py index eb2522536459..d89b82e9472f 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -1389,6 +1389,29 @@ def test_upload_data_with_missing_schema_fields_raises_error(self): private_key=self.credentials, table_schema=test_schema) + def test_upload_data_with_timestamp(self): + test_id = "21" + test_size = 6 + df = DataFrame(np.random.randn(test_size, 4), index=range(test_size), + columns=list('ABCD')) + df['times'] = np.datetime64('2018-03-13T05:40:45.348318Z') + + gbq.to_gbq( + df, self.destination_table + test_id, + _get_project_id(), + private_key=self.credentials) + + result_df = gbq.read_gbq("SELECT * FROM {0}".format( + self.destination_table + test_id), + project_id=_get_project_id(), + private_key=self.credentials) + + assert len(result_df) == test_size + + expected = df['times'].sort_values() + result = result_df['times'].sort_values() + tm.assert_numpy_array_equal(expected.values, result.values) + def test_list_dataset(self): dataset_id = self.dataset_prefix + "1" assert dataset_id in self.dataset.datasets() From 362581627a9158dd294018775a6d8fd21d911332 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Wed, 14 Mar 2018 20:20:49 +0000 Subject: [PATCH 112/519] Load data with user defined schema (#150) * Load data with the supplied schema * Test for upload data with different df and user schema --- packages/pandas-gbq/pandas_gbq/gbq.py | 2 +- .../pandas-gbq/pandas_gbq/tests/test_gbq.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 606f2810ca31..e4d0304383f1 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -562,7 +562,7 @@ def load_data( try: for remaining_rows in _load.load_chunks( self.client, dataframe, dataset_id, table_id, - chunksize=chunksize): + chunksize=chunksize, schema=schema): self._print("\rLoad is {0}% Complete".format( ((total_rows - remaining_rows) * 100) / total_rows)) except self.http_error as ex: diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py index d89b82e9472f..2214ebaaf741 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -1412,6 +1412,23 @@ def test_upload_data_with_timestamp(self): result = result_df['times'].sort_values() tm.assert_numpy_array_equal(expected.values, result.values) + def test_upload_data_with_different_df_and_user_schema(self): + df = tm.makeMixedDataFrame() + df['A'] = df['A'].astype(str) + df['B'] = df['B'].astype(str) + test_id = "22" + test_schema = [{'name': 'A', 'type': 'FLOAT'}, + {'name': 'B', 'type': 'FLOAT'}, + {'name': 'C', 'type': 'STRING'}, + {'name': 'D', 'type': 'TIMESTAMP'}] + destination_table = self.destination_table + test_id + gbq.to_gbq(df, destination_table, _get_project_id(), + private_key=self.credentials, + table_schema=test_schema) + dataset, table = destination_table.split('.') + assert self.table.verify_schema(dataset, table, + dict(fields=test_schema)) + def test_list_dataset(self): dataset_id = self.dataset_prefix + "1" assert dataset_id in self.dataset.datasets() From dda65e58c70334d39971bf5ab1a510aea570fa17 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+maxim-lian@users.noreply.github.com> Date: Wed, 21 Mar 2018 16:43:58 -0400 Subject: [PATCH 113/519] fix & parametrize (#147) --- .../pandas-gbq/pandas_gbq/tests/test_gbq.py | 58 ++++++------------- 1 file changed, 18 insertions(+), 40 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py index 2214ebaaf741..2df1b9bde935 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -280,53 +280,31 @@ def test_get_user_account_credentials_returns_credentials(self): assert isinstance(credentials, Credentials) -@pytest.mark.skip('Currently fails, see ' - 'https://github.com/pydata/pandas-gbq/pull/125') class TestGBQUnit(object): - def test_import_google_api_python_client(self): - if not _in_travis_environment(): - pytest.skip("Skip if not in travis environment. Extra test to " - "make sure pandas_gbq doesn't break when " - "using google-api-python-client==1.2") - - if compat.PY2: - with pytest.raises(ImportError): - from googleapiclient.discovery import build # noqa - from googleapiclient.errors import HttpError # noqa - from apiclient.discovery import build # noqa - from apiclient.errors import HttpError # noqa - else: - from googleapiclient.discovery import build # noqa - from googleapiclient.errors import HttpError # noqa - def test_should_return_credentials_path_set_by_env_var(self): env = {'PANDAS_GBQ_CREDENTIALS_FILE': '/tmp/dummy.dat'} with mock.patch.dict('os.environ', env): assert gbq._get_credentials_file() == '/tmp/dummy.dat' - assert gbq._get_credentials_file() == 'bigquery_credentials.dat' - - def test_should_return_bigquery_integers_as_python_ints(self): - result = gbq._parse_entry(1, 'INTEGER') - assert result == int(1) - - def test_should_return_bigquery_floats_as_python_floats(self): - result = gbq._parse_entry(1, 'FLOAT') - assert result == float(1) - - def test_should_return_bigquery_timestamps_as_numpy_datetime(self): - result = gbq._parse_entry('0e9', 'TIMESTAMP') - assert result == np_datetime64_compat('1970-01-01T00:00:00Z') - - def test_should_return_bigquery_booleans_as_python_booleans(self): - result = gbq._parse_entry('false', 'BOOLEAN') - assert not result - - def test_should_return_bigquery_strings_as_python_strings(self): - result = gbq._parse_entry('STRING', 'STRING') - assert result == 'STRING' + @pytest.mark.parametrize( + ('input', 'type_', 'expected'), [ + (1, 'INTEGER', int(1)), + (1, 'FLOAT', float(1)), + pytest.param('false', 'BOOLEAN', False, marks=pytest.mark.xfail), + pytest.param( + '0e9', 'TIMESTAMP', + np_datetime64_compat('1970-01-01T00:00:00Z'), + marks=pytest.mark.xfail), + ('STRING', 'STRING', 'STRING'), + ]) + def test_should_return_bigquery_correctly_typed( + self, input, type_, expected): + result = gbq._parse_data( + dict(fields=[dict(name='x', type=type_, mode='NULLABLE')]), + rows=[[input]]).iloc[0, 0] + assert result == expected def test_to_gbq_should_fail_if_invalid_table_name_passed(self): with pytest.raises(gbq.NotFoundException): @@ -893,7 +871,7 @@ def test_array_of_floats(self): private_key=_get_private_key_path(), dialect='standard') tm.assert_frame_equal(df, DataFrame([[[1.1, 2.2, 3.3], 4]], - columns=["a", "b"])) + columns=["a", "b"])) class TestToGBQIntegration(object): From c820feaba6e14e1ee4a9830c65d0bb52f074906b Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+maxim-lian@users.noreply.github.com> Date: Wed, 21 Mar 2018 18:34:35 -0400 Subject: [PATCH 114/519] Remove scripts (#146) * remove merge-py * repo ops --- packages/pandas-gbq/.gitignore | 1 + packages/pandas-gbq/.stickler.yml | 4 + packages/pandas-gbq/scripts/merge-py.py | 264 ------------------------ 3 files changed, 5 insertions(+), 264 deletions(-) delete mode 100755 packages/pandas-gbq/scripts/merge-py.py diff --git a/packages/pandas-gbq/.gitignore b/packages/pandas-gbq/.gitignore index 104e28dfa1aa..147e7e1e5bee 100644 --- a/packages/pandas-gbq/.gitignore +++ b/packages/pandas-gbq/.gitignore @@ -46,6 +46,7 @@ coverage_html_report *.so .build_cache_dir MANIFEST +__pycache__ # Python files # ################ diff --git a/packages/pandas-gbq/.stickler.yml b/packages/pandas-gbq/.stickler.yml index 1a31b479c0ca..17cd4582f9e7 100644 --- a/packages/pandas-gbq/.stickler.yml +++ b/packages/pandas-gbq/.stickler.yml @@ -2,5 +2,9 @@ linters: flake8: max-line-length: 79 fixer: true + ignore: +files: + ignore: + - doc/**/*.py fixers: enable: true diff --git a/packages/pandas-gbq/scripts/merge-py.py b/packages/pandas-gbq/scripts/merge-py.py deleted file mode 100755 index 062ca6880f0c..000000000000 --- a/packages/pandas-gbq/scripts/merge-py.py +++ /dev/null @@ -1,264 +0,0 @@ -#!/usr/bin/env python - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Utility for creating well-formed pull request merges and pushing them to -# Apache. -# usage: ./apache-pr-merge.py (see config env vars below) -# -# Lightly modified from version of this script in incubator-parquet-format - -from __future__ import print_function - -import os -import sys -import textwrap -from subprocess import check_output - -import requests -import six -from requests.auth import HTTPBasicAuth -from six.moves import input - -PANDASGBQ_HOME = '.' -PROJECT_NAME = 'pandas-gbq' -print("PANDASGBQ_HOME = " + PANDASGBQ_HOME) - -# Remote name with the PR -PR_REMOTE_NAME = os.environ.get("PR_REMOTE_NAME", "upstream") - -# Remote name where results pushed -PUSH_REMOTE_NAME = os.environ.get("PUSH_REMOTE_NAME", "upstream") - -GITHUB_BASE = "https://github.com/pydata/" + PROJECT_NAME + "/pull" -GITHUB_API_BASE = "https://api.github.com/repos/pydata/" + PROJECT_NAME - -# Prefix added to temporary branches -BRANCH_PREFIX = "PR_TOOL" - -os.chdir(PANDASGBQ_HOME) - -auth_required = False - -if auth_required: - GITHUB_USERNAME = os.environ['GITHUB_USER'] - import getpass - GITHUB_PASSWORD = getpass.getpass('Enter github.com password for %s:' - % GITHUB_USERNAME) - - def get_json_auth(url): - auth = HTTPBasicAuth(GITHUB_USERNAME, GITHUB_PASSWORD) - req = requests.get(url, auth=auth) - return req.json() - - get_json = get_json_auth -else: - def get_json_no_auth(url): - req = requests.get(url) - return req.json() - - get_json = get_json_no_auth - - -def fail(msg): - print(msg) - clean_up() - sys.exit(-1) - - -def run_cmd(cmd): - if isinstance(cmd, six.string_types): - cmd = cmd.split(' ') - - output = check_output(cmd) - - if isinstance(output, six.binary_type): - output = output.decode('utf-8') - return output - - -def continue_maybe(prompt): - result = input("\n%s (y/n): " % prompt) - if result.lower() != "y": - fail("Okay, exiting") - - -original_head = run_cmd("git rev-parse HEAD")[:8] - - -def clean_up(): - print("Restoring head pointer to %s" % original_head) - run_cmd("git checkout %s" % original_head) - - branches = run_cmd("git branch").replace(" ", "").split("\n") - - for branch in [b for b in branches if b.startswith(BRANCH_PREFIX)]: - print("Deleting local branch %s" % branch) - run_cmd("git branch -D %s" % branch) - - -# Merge the requested PR and return the merge hash -def merge_pr(pr_num, target_ref): - - pr_branch_name = "%s_MERGE_PR_%s" % (BRANCH_PREFIX, pr_num) - target_branch_name = "%s_MERGE_PR_%s_%s" % (BRANCH_PREFIX, pr_num, - target_ref.upper()) - run_cmd("git fetch %s pull/%s/head:%s" % (PR_REMOTE_NAME, pr_num, - pr_branch_name)) - run_cmd("git fetch %s %s:%s" % (PUSH_REMOTE_NAME, target_ref, - target_branch_name)) - run_cmd("git checkout %s" % target_branch_name) - - had_conflicts = False - try: - run_cmd(['git', 'merge', pr_branch_name, '--squash']) - except Exception as e: - msg = ("Error merging: %s\nWould you like to manually fix-up " - "this merge?" % e) - continue_maybe(msg) - msg = ("Okay, please fix any conflicts and 'git add' " - "conflicting files... Finished?") - continue_maybe(msg) - had_conflicts = True - - commit_authors = run_cmd(['git', 'log', 'HEAD..%s' % pr_branch_name, - '--pretty=format:%an <%ae>']).split("\n") - distinct_authors = sorted(set(commit_authors), - key=lambda x: commit_authors.count(x), - reverse=True) - primary_author = distinct_authors[0] - commits = run_cmd(['git', 'log', 'HEAD..%s' % pr_branch_name, - '--pretty=format:%h [%an] %s']).split("\n\n") - - merge_message_flags = [] - - merge_message_flags += ["-m", title] - if body is not None: - merge_message_flags += ["-m", '\n'.join(textwrap.wrap(body))] - - authors = "\n".join(["Author: %s" % a for a in distinct_authors]) - - merge_message_flags += ["-m", authors] - - if had_conflicts: - committer_name = run_cmd("git config --get user.name").strip() - committer_email = run_cmd("git config --get user.email").strip() - message = ("This patch had conflicts when merged, " - "resolved by\nCommitter: %s <%s>" - % (committer_name, committer_email)) - merge_message_flags += ["-m", message] - - # The string "Closes #%s" string is required for GitHub to correctly close - # the PR - merge_message_flags += [ - "-m", - "Closes #%s from %s and squashes the following commits:" - % (pr_num, pr_repo_desc)] - for c in commits: - merge_message_flags += ["-m", c] - - run_cmd(['git', 'commit', '--author="%s"' % primary_author] + - merge_message_flags) - - continue_maybe("Merge complete (local ref %s). Push to %s?" % ( - target_branch_name, PUSH_REMOTE_NAME)) - - try: - run_cmd('git push %s %s:%s' % (PUSH_REMOTE_NAME, target_branch_name, - target_ref)) - except Exception as e: - clean_up() - fail("Exception while pushing: %s" % e) - - merge_hash = run_cmd("git rev-parse %s" % target_branch_name)[:8] - clean_up() - print("Pull request #%s merged!" % pr_num) - print("Merge hash: %s" % merge_hash) - return merge_hash - - -def cherry_pick(pr_num, merge_hash, default_branch): - pick_ref = input("Enter a branch name [%s]: " % default_branch) - if pick_ref == "": - pick_ref = default_branch - - pick_branch_name = "%s_PICK_PR_%s_%s" % (BRANCH_PREFIX, pr_num, - pick_ref.upper()) - - run_cmd("git fetch %s %s:%s" % (PUSH_REMOTE_NAME, pick_ref, - pick_branch_name)) - run_cmd("git checkout %s" % pick_branch_name) - run_cmd("git cherry-pick -sx %s" % merge_hash) - - continue_maybe("Pick complete (local ref %s). Push to %s?" % ( - pick_branch_name, PUSH_REMOTE_NAME)) - - try: - run_cmd('git push %s %s:%s' % (PUSH_REMOTE_NAME, pick_branch_name, - pick_ref)) - except Exception as e: - clean_up() - fail("Exception while pushing: %s" % e) - - pick_hash = run_cmd("git rev-parse %s" % pick_branch_name)[:8] - clean_up() - - print("Pull request #%s picked into %s!" % (pr_num, pick_ref)) - print("Pick hash: %s" % pick_hash) - return pick_ref - - -def fix_version_from_branch(branch, versions): - # Note: Assumes this is a sorted (newest->oldest) list of un-released - # versions - if branch == "master": - return versions[0] - else: - branch_ver = branch.replace("branch-", "") - return filter(lambda x: x.name.startswith(branch_ver), versions)[-1] - - -pr_num = input("Which pull request would you like to merge? (e.g. 34): ") -pr = get_json("%s/pulls/%s" % (GITHUB_API_BASE, pr_num)) - -url = pr["url"] -title = pr["title"] -body = pr["body"] -target_ref = pr["base"]["ref"] -user_login = pr["user"]["login"] -base_ref = pr["head"]["ref"] -pr_repo_desc = "%s/%s" % (user_login, base_ref) - -if pr["merged"] is True: - print("Pull request {0} has already been merged, please backport manually" - .format(pr_num)) - sys.exit(0) - -if not bool(pr["mergeable"]): - msg = ("Pull request {0} is not mergeable in its current form.\n" - "Continue? (experts only!)".format(pr_num)) - continue_maybe(msg) - -print("\n=== Pull Request #%s ===" % pr_num) -print("title\t%s\nsource\t%s\ntarget\t%s\nurl\t%s" - % (title, pr_repo_desc, target_ref, url)) -continue_maybe("Proceed with merging pull request #%s?" % pr_num) - -merged_refs = [target_ref] - -merge_hash = merge_pr(pr_num, target_ref) From c345a980884f75ec12806654b822dee0b16b0397 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+maxim-lian@users.noreply.github.com> Date: Fri, 23 Mar 2018 21:31:42 -0400 Subject: [PATCH 115/519] ENH: log rather than print (#18) * rebase * Changes from feedback * remove verbose from docstrings and internal classes * WIP * update for newer code * remove verbose for private class * updates if cache hit * couple refinements * remove commented out code * show logs in pytest * docs --- packages/pandas-gbq/.travis.yml | 2 +- packages/pandas-gbq/docs/source/intro.rst | 21 +++- packages/pandas-gbq/docs/source/reading.rst | 4 - packages/pandas-gbq/docs/source/writing.rst | 18 ---- packages/pandas-gbq/pandas_gbq/gbq.py | 102 ++++++++++---------- 5 files changed, 71 insertions(+), 76 deletions(-) diff --git a/packages/pandas-gbq/.travis.yml b/packages/pandas-gbq/.travis.yml index 81417d973633..92129cc6fee5 100644 --- a/packages/pandas-gbq/.travis.yml +++ b/packages/pandas-gbq/.travis.yml @@ -42,6 +42,6 @@ install: - python setup.py install script: - - pytest -s -v --cov=pandas_gbq --cov-report xml:/tmp/pytest-cov.xml pandas_gbq + - pytest -v --cov=pandas_gbq --cov-report xml:/tmp/pytest-cov.xml pandas_gbq - if [[ $COVERAGE == 'true' ]]; then codecov ; fi - if [[ $LINT == 'true' ]]; then flake8 pandas_gbq -v ; fi diff --git a/packages/pandas-gbq/docs/source/intro.rst b/packages/pandas-gbq/docs/source/intro.rst index 3cac399a6ddc..0cad9ae8932f 100644 --- a/packages/pandas-gbq/docs/source/intro.rst +++ b/packages/pandas-gbq/docs/source/intro.rst @@ -26,6 +26,23 @@ While this trade-off works well for most cases, it breaks down for storing values greater than 2**53. Such values in BigQuery can represent identifiers and unnoticed precision lost for identifier is what we want to avoid. +Logging ++++++++ + +Because some requests take some time, this library will log its progress of +longer queries. IPython & Jupyter by default attach a handler to the logger. +If you're running in another process and want to see logs, or you want to see +more verbose logs, you can do something like: + +.. code-block:: ipython + + import logging + import sys + logger = logging.getLogger('pandas_gbq') + logger.setLevel(logging.DEBUG) + logger.addHandler(logging.StreamHandler(stream=sys.stdout)) + + .. _authentication: Authentication @@ -49,8 +66,8 @@ Additional information on service accounts can be found `here `__. Authentication via ``application default credentials`` is also possible, but only valid -if the parameter ``private_key`` is not provided. This method requires that the -credentials can be fetched from the development environment. Otherwise, the OAuth2 +if the parameter ``private_key`` is not provided. This method requires that the +credentials can be fetched from the development environment. Otherwise, the OAuth2 client-side authentication is used. Additional information can be found on `application default credentials `__. diff --git a/packages/pandas-gbq/docs/source/reading.rst b/packages/pandas-gbq/docs/source/reading.rst index ebfcfa9ba345..16abbdc32dd8 100644 --- a/packages/pandas-gbq/docs/source/reading.rst +++ b/packages/pandas-gbq/docs/source/reading.rst @@ -45,10 +45,6 @@ For more information about query configuration parameters see You can find your project id in the `Google developers console `__. -.. note:: - - You can toggle the verbose output via the ``verbose`` flag which defaults to ``True``. - .. note:: The ``dialect`` argument can be used to indicate whether to use BigQuery's ``'legacy'`` SQL diff --git a/packages/pandas-gbq/docs/source/writing.rst b/packages/pandas-gbq/docs/source/writing.rst index 2a30bc3550f8..7a54a3622003 100644 --- a/packages/pandas-gbq/docs/source/writing.rst +++ b/packages/pandas-gbq/docs/source/writing.rst @@ -54,24 +54,6 @@ For example, the following writes ``df`` to a BigQuery table in batches of 10000 to_gbq(df, 'my_dataset.my_table', projectid, chunksize=10000) -You can also see the progress of your post via the ``verbose`` flag which defaults to ``True``. -For example: - -.. code-block:: python - - In [8]: to_gbq(df, 'my_dataset.my_table', projectid, chunksize=10000, verbose=True) - - Streaming Insert is 10% Complete - Streaming Insert is 20% Complete - Streaming Insert is 30% Complete - Streaming Insert is 40% Complete - Streaming Insert is 50% Complete - Streaming Insert is 60% Complete - Streaming Insert is 70% Complete - Streaming Insert is 80% Complete - Streaming Insert is 90% Complete - Streaming Insert is 100% Complete - .. note:: If an error occurs while streaming data to BigQuery, see diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index e4d0304383f1..6d5aacf85106 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -1,6 +1,6 @@ import json +import logging import os -import sys import time import warnings from datetime import datetime @@ -11,6 +11,8 @@ from pandas import DataFrame, compat from pandas.compat import lzip +logger = logging.getLogger(__name__) + def _check_google_client_version(): @@ -162,7 +164,7 @@ class TableCreationError(ValueError): class GbqConnector(object): scope = 'https://www.googleapis.com/auth/bigquery' - def __init__(self, project_id, reauth=False, verbose=False, + def __init__(self, project_id, reauth=False, private_key=None, auth_local_webserver=False, dialect='legacy'): from google.api_core.exceptions import GoogleAPIError @@ -170,7 +172,6 @@ def __init__(self, project_id, reauth=False, verbose=False, self.http_error = (ClientError, GoogleAPIError) self.project_id = project_id self.reauth = reauth - self.verbose = verbose self.private_key = private_key self.auth_local_webserver = auth_local_webserver self.dialect = dialect @@ -324,7 +325,7 @@ def save_user_account_credentials(self, credentials): } json.dump(credentials_json, credentials_file) except IOError: - self._print('Unable to save credentials.') + logger.warning('Unable to save credentials.') def get_user_account_credentials(self): """Gets user account credentials. @@ -410,22 +411,17 @@ def get_service_account_credentials(self): "Can be obtained from: https://console.developers.google." "com/permissions/serviceaccounts") - def _print(self, msg, end='\n'): - if self.verbose: - sys.stdout.write(msg + end) - sys.stdout.flush() - def _start_timer(self): self.start = time.time() def get_elapsed_seconds(self): return round(time.time() - self.start, 2) - def print_elapsed_seconds(self, prefix='Elapsed', postfix='s.', - overlong=7): + def log_elapsed_seconds(self, prefix='Elapsed', postfix='s.', + overlong=7): sec = self.get_elapsed_seconds() if sec > overlong: - self._print('{} {} {}'.format(prefix, sec, postfix)) + logger.info('{} {} {}'.format(prefix, sec, postfix)) # http://stackoverflow.com/questions/1094841/reusable-library-to-get-human-readable-version-of-file-size @staticmethod @@ -481,11 +477,12 @@ def run_query(self, query, **kwargs): self._start_timer() try: - self._print('Requesting query... ', end="") + + logger.info('Requesting query... ') query_reply = self.client.query( query, job_config=QueryJobConfig.from_api_repr(job_config['query'])) - self._print('ok.') + logger.info('ok.\nQuery running...') except (RefreshError, ValueError): if self.private_key: raise AccessDenied( @@ -498,10 +495,10 @@ def run_query(self, query, **kwargs): self.process_http_error(ex) job_id = query_reply.job_id - self._print('Job ID: %s\nQuery running...' % job_id) + logger.info('Job ID: %s\nQuery running...' % job_id) while query_reply.state != 'DONE': - self.print_elapsed_seconds(' Elapsed', 's. Waiting...') + self.log_elapsed_seconds(' Elapsed', 's. Waiting...') timeout_ms = job_config['query'].get('timeoutMs') if timeout_ms and timeout_ms < self.get_elapsed_seconds() * 1000: @@ -520,19 +517,16 @@ def run_query(self, query, **kwargs): except self.http_error as ex: self.process_http_error(ex) - if self.verbose: - if query_reply.cache_hit: - self._print('Query done.\nCache hit.\n') - else: - bytes_processed = query_reply.total_bytes_processed or 0 - bytes_billed = query_reply.total_bytes_billed or 0 - self._print('Query done.\nProcessed: {} Billed: {}'.format( - self.sizeof_fmt(bytes_processed), - self.sizeof_fmt(bytes_billed))) - self._print('Standard price: ${:,.2f} USD\n'.format( - bytes_billed * self.query_price_for_TB)) - - self._print('Retrieving results...') + if query_reply.cache_hit: + logger.debug('Query done.\nCache hit.\n') + else: + bytes_processed = query_reply.total_bytes_processed or 0 + bytes_billed = query_reply.total_bytes_billed or 0 + logger.debug('Query done.\nProcessed: {} Billed: {}'.format( + self.sizeof_fmt(bytes_processed), + self.sizeof_fmt(bytes_billed))) + logger.debug('Standard price: ${:,.2f} USD\n'.format( + bytes_billed * self.query_price_for_TB)) try: rows_iter = query_reply.result() @@ -546,8 +540,8 @@ def run_query(self, query, **kwargs): for field in rows_iter.schema], } - # print basic query stats - self._print('Got {} rows.\n'.format(total_rows)) + # log basic query stats + logger.info('Got {} rows.\n'.format(total_rows)) return schema, result_rows @@ -557,18 +551,18 @@ def load_data( from pandas_gbq import _load total_rows = len(dataframe) - self._print("\n\n") + logger.info("\n\n") try: for remaining_rows in _load.load_chunks( self.client, dataframe, dataset_id, table_id, chunksize=chunksize, schema=schema): - self._print("\rLoad is {0}% Complete".format( + logger.info("\rLoad is {0}% Complete".format( ((total_rows - remaining_rows) * 100) / total_rows)) except self.http_error as ex: self.process_http_error(ex) - self._print("\n") + logger.info("\n") def schema(self, dataset_id, table_id): """Retrieve the schema of the table @@ -687,7 +681,7 @@ def delete_and_recreate_table(self, dataset_id, table_id, table_schema): # be a 120 second delay if not self.verify_schema(dataset_id, table_id, table_schema): - self._print('The existing table has a different schema. Please ' + logger.info('The existing table has a different schema. Please ' 'wait 2 minutes. See Google BigQuery issue #191') delay = 120 @@ -729,7 +723,7 @@ def _parse_data(schema, rows): def read_gbq(query, project_id=None, index_col=None, col_order=None, - reauth=False, verbose=True, private_key=None, + reauth=False, verbose=None, private_key=None, auth_local_webserver=False, dialect='legacy', **kwargs): r"""Load data from Google BigQuery using google-cloud-python @@ -768,8 +762,6 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, reauth : boolean (default False) Force Google BigQuery to reauthenticate the user. This is useful if multiple accounts are used. - verbose : boolean (default True) - Verbose output private_key : str (optional) Service account private key in JSON format. Can be file path or string contents. This is useful for remote server @@ -793,6 +785,7 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, compliant with the SQL 2011 standard. For more information see `BigQuery SQL Reference `__ + verbose : None, deprecated **kwargs : Arbitrary keyword arguments configuration (dict): query config parameters for job processing. @@ -809,6 +802,11 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, DataFrame representing results of query """ + if verbose is not None: + warnings.warn( + "verbose is deprecated and will be removed in " + "a future version. Set logging level in order to vary " + "verbosity", FutureWarning, stacklevel=1) _test_google_api_imports() @@ -819,7 +817,7 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, raise ValueError("'{0}' is not valid for dialect".format(dialect)) connector = GbqConnector( - project_id, reauth=reauth, verbose=verbose, private_key=private_key, + project_id, reauth=reauth, private_key=private_key, dialect=dialect, auth_local_webserver=auth_local_webserver) schema, rows = connector.run_query(query, **kwargs) final_df = _parse_data(schema, rows) @@ -853,7 +851,7 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, final_df[field['name']] = \ final_df[field['name']].astype(type_map[field['type'].upper()]) - connector.print_elapsed_seconds( + connector.log_elapsed_seconds( 'Total time taken', datetime.now().strftime('s.\nFinished at %Y-%m-%d %H:%M:%S.'), 0 @@ -863,7 +861,7 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, def to_gbq(dataframe, destination_table, project_id, chunksize=None, - verbose=True, reauth=False, if_exists='fail', private_key=None, + verbose=None, reauth=False, if_exists='fail', private_key=None, auth_local_webserver=False, table_schema=None): """Write a DataFrame to a Google BigQuery table. @@ -899,8 +897,6 @@ def to_gbq(dataframe, destination_table, project_id, chunksize=None, chunksize : int (default None) Number of rows to be inserted in each chunk from the dataframe. Use ``None`` to load the dataframe in a single chunk. - verbose : boolean (default True) - Show percentage complete reauth : boolean (default False) Force Google BigQuery to reauthenticate the user. This is useful if multiple accounts are used. @@ -930,10 +926,17 @@ def to_gbq(dataframe, destination_table, project_id, chunksize=None, of DataFrame columns. See BigQuery API documentation on available names of a field. .. versionadded:: 0.3.1 + verbose : None, deprecated """ _test_google_api_imports() + if verbose is not None: + warnings.warn( + "verbose is deprecated and will be removed in " + "a future version. Set logging level in order to vary " + "verbosity", FutureWarning, stacklevel=1) + if if_exists not in ('fail', 'replace', 'append'): raise ValueError("'{0}' is not valid for if_exists".format(if_exists)) @@ -942,7 +945,7 @@ def to_gbq(dataframe, destination_table, project_id, chunksize=None, "Invalid Table Name. Should be of the form 'datasetId.tableId' ") connector = GbqConnector( - project_id, reauth=reauth, verbose=verbose, private_key=private_key, + project_id, reauth=reauth, private_key=private_key, auth_local_webserver=auth_local_webserver) dataset_id, table_id = destination_table.rsplit('.', 1) @@ -1004,10 +1007,9 @@ def _generate_bq_schema(df, default_type='STRING'): class _Table(GbqConnector): - def __init__(self, project_id, dataset_id, reauth=False, verbose=False, - private_key=None): + def __init__(self, project_id, dataset_id, reauth=False, private_key=None): self.dataset_id = dataset_id - super(_Table, self).__init__(project_id, reauth, verbose, private_key) + super(_Table, self).__init__(project_id, reauth, private_key) def exists(self, table_id): """ Check if a table exists in Google BigQuery @@ -1101,10 +1103,8 @@ def delete(self, table_id): class _Dataset(GbqConnector): - def __init__(self, project_id, reauth=False, verbose=False, - private_key=None): - super(_Dataset, self).__init__(project_id, reauth, verbose, - private_key) + def __init__(self, project_id, reauth=False, private_key=None): + super(_Dataset, self).__init__(project_id, reauth, private_key) def exists(self, dataset_id): """ Check if a dataset exists in Google BigQuery From 01708e332c504a1c15fa1889252458baead6b95c Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+maxim-lian@users.noreply.github.com> Date: Sun, 1 Apr 2018 07:56:02 -0400 Subject: [PATCH 116/519] bump google auth version (#143) * bump google auth verison to 1.0.2 * bump to 1.4.1 * remove pins --- packages/pandas-gbq/AUTHORS.md | 1 - packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip | 2 +- packages/pandas-gbq/setup.py | 4 ++-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/pandas-gbq/AUTHORS.md b/packages/pandas-gbq/AUTHORS.md index dcaaea101f4c..152c93a42360 100644 --- a/packages/pandas-gbq/AUTHORS.md +++ b/packages/pandas-gbq/AUTHORS.md @@ -54,4 +54,3 @@ pandas is distributed under a 3-clause ("Simplified" or "New") BSD license. Parts of NumPy, SciPy, numpydoc, bottleneck, which all have BSD-compatible licenses, are included. Their licenses follow the pandas license. - diff --git a/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip b/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip index f895fb1f1a62..68ac370dc0d4 100644 --- a/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip +++ b/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip @@ -1,4 +1,4 @@ -google-auth==1.0.2 +google-auth==1.4.1 google-auth-oauthlib==0.0.1 mock google-cloud-bigquery==0.29.0 diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index a240cf45fa56..d3c353118b0b 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -18,8 +18,8 @@ def readme(): INSTALL_REQUIRES = [ 'pandas', - 'google-auth>=1.0.0', - 'google-auth-oauthlib>=0.0.1', + 'google-auth', + 'google-auth-oauthlib', 'google-cloud-bigquery>=0.29.0', ] From 899c31ab3e7ed3388951cc7ddcf6e9eec62c2ff6 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+maxim-lian@users.noreply.github.com> Date: Mon, 2 Apr 2018 17:10:04 -0400 Subject: [PATCH 117/519] remove prof files (#153) --- packages/pandas-gbq/prof/combined.prof | Bin 206840 -> 0 bytes .../prof/test_upload_chinese_unicode_data.prof | Bin 206840 -> 0 bytes 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 packages/pandas-gbq/prof/combined.prof delete mode 100644 packages/pandas-gbq/prof/test_upload_chinese_unicode_data.prof diff --git a/packages/pandas-gbq/prof/combined.prof b/packages/pandas-gbq/prof/combined.prof deleted file mode 100644 index d307f0720732b8ae4cad63b1b0d3eb4e564ffc7f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 206840 zcmc${37nio(LWwYHpk{BB#>|=Tp>WRIUocGJKTpH2?6A0nBARy@+7-6L(gokMG#N~ zQ4U2wL@q(i3%mjXA~$(OL=m}>K{N=811MJj2_V1kx9jQd=b4@zHvC!t`}y$D*{$xb zuCA`GuBxv7XiERJ>z!8Vho2Q&&+W{X7UtU~PHxM#=G(H9C+6EHPHtIV$`(;+YJpmaTDK)9H_@DuF6$G$(>{hZE}Ev)Rn zcF%S0q)T<^UNfqgWp~^5M|Y!6>B^U~>2{UxOzVM88Bo~~v`~Zb2~syadhxS=SwCe9 z=s2~`9WN}E3QesYVK!Gnzm-*Jxjei;G|~&RnKlpxosMUx!`bQPGgjoT*)dHJhZziWL9r=p%VpT_o$O%%KIm;I%cI%b!p)94bX3=y&9;Ev!d~=-IW0C7GP$-) zacZ%YY3ayT&1o?#Wv3Q0t&1~3w%8Qp^8r9IU8RLh7Hk77)3O(xv?3Y%ZP6wH5MVu9Q~UQ@XOn5~xUG zs!fRLk*|z8rMUg`&9*RoN`IvX0-o--c1ydk7NLormet0n5bgx74Bi>A>abfD?U@RW zJ7|adu6gU+RJR&}cBTx_93pIT_wxVQCwKM^slXIg_E(Jn3XkM6YD^b1?OEWOX~R<5 z6u;7+;EltUUb*bCaVc9cwLau(7?v(#nCX01snCT1#(HZ%f$a0poYTiN zVj#25UGd;gc0D)MH#o)lebG|A?o%L1d7xgGS&)eWMnSsu3*K7z@;SHv`;}%}07}6Z zPo~21Offej4I*`QU_}%E&7~~+)5@?Btc-)AH$D0!iZ^c%2 z;i*Mk#ZnI|FE&hRZ?$P}b#phy+0r-TEU8Sch=J*OxQJsN0{JpC+J@>Ol^mqowpFpD z!a@(Dtn7>~)Es<*hHEEh7wj`MWeeS^0lx=j`R5*BlASur77HC=Pm~Rhq)g){xb=+E zE#dXkQ?>xgDso@tyX5$WQy6@J8=!=(Q0`#MydCMmf1TQ0i^ruq0_`b z)^2MD!rei}5M*;%l_}*F{%IA%3Z!m7wVxXY)}`zhGg4eft-Tn>I_edLj;RLJd-CuR?fp}Uk;cSDrckhB?FhBxu)_>D9l`As zpWwc;SKs-qAABig3moR0UJNriU9I`Ta!%Hh#C=4%Et@UyS84W)tzrY>e;7Vi5wZoW zp@Z><%EzoZG`$!#)0vKN33r%jtVSIslG*PV;r&jt+8@6L2(=S+1VQkh{;`EA{nuuC zF_3kfuQZf?C{InDnJ|YDhhw(#^-tIN_howg80o?FF%tvz`SJI-1(E**1L`Q0%bG z4u@dN!|4EP4D#k7>Tdi5n{K`Dhi8u4GGz;x{1X6Qd5D#UrE}S(>CSAi2!$Ku<5@y9 z#Re_yUvG}2{p;Mi*IlF(p&DeV$b`>w&V~lr5~fXsx%ND0Lq$eq@v-`WEda}b_(M3C zI@u6NA|rt+SV8^NLRJ#}`4OW(Khg~D+v{@@G$H{p{R_l&c~CJ6r9HDWK}%0|gC?b3 zbhwV}URWG}v5WaG)e4m>y#z8PuW<0?{0fVOSCX2;)0yHC*t>JVA+qGD$*UYmQ|l0v zlrTlrYEEK7%PD>_&EMibQe--N8`6=7N`=iWgPG&pDAhwxs(?R)Z*bUasaD_J0ofoVYP+ z4bLuvj?`L0<+kjy-c{TQ78C4v3-g_MFlM-dGLL*1XU3b29(6NiULTka!~i0MfbO1E zxr`h@x51uW9!ljt-J%awHLyze{4)x7Mf_65GmQOUo`vmi{J*e4gOHsgW!V;nbN{ET11`6 z_=UQhb-a@x22$(=xz=$hMTUabj(k^JQ%e|tzG`{GM7D-FZkSg2rRh>vp(8R@E-F$< zs%%btud0c!;#j~tG*HPI-rHprO`T|kd$771A8UfJ1&q26PF-S)Xeuo)WShEj;S$P^ z#|LAaa}rI!%QlNPY2W)@C4X?jR*`y^pw#6_qlbX@w$EgfSD! zMwa+G?f_}^PfW6`J>|cJujmt$E7t&t^0)|H`L$J{z^WVJ&+2FR)Y|LRjrc?59%-Sa zE9_{C6qQ&D^)`vbX>hGGafncrlW+BDu(A^|x{Vl#RxmgfeB3$j!ax3N1kAQu;P;@i zQ^q}GpQ7vph!fPp--*3Tq}2_8EDu-N06Jn;Ve88;PPA>^6~cN$s!q2P?K!@XkBmg# zU}m8UMNfg%S8Rdy8hzZLnqMAJ=qf_@i0w=s+E9)NuMQ#nurxZYc|3XK;eV~aO~d=m zfhnNg`KVHPPxk}0GikxScH6b5UOf8M=D-vZv$n$7+FSEoxyUe8Jx-4aV5t!NAJ-O_ zq=F5$gjBEZ!s_Y@+sM9)i1gsJs6lBEs9Gn9^^`OtO4Z#EHB%*T*<|tC9PT3;#Mi6_^5jfa~tC5-%B8I?OK0 zXd(L%T?@mp!}E9POsPaiS-p;v!`O5#B|vFvn41{6HJzIYUd+l=5K29aPjK-cZ@>4& z?`)c~g>H2}exH~;&;E@;_Ju;Jjr33`JE4>`QKbyB+Awf3v8AF98MqzGzr`omai2|J z|Lg1tDO*U8VC{BxCevI-!U{5>)X-6KMy*eVf^YZ<_PTTZ#`o6DNCi)R^y}xYJa%RZ z6Dmje-EhoV+c=jjNer8DII2@Efy)XfeSk?1I2<|%IPeopI43*n8`tfhvIVmC!5DJo zYxtB08xt8YXaiFE6ZHSy+i%XAwqwc`u;ccDohY(7nl;W<|BOazZbzoGr7Z)p#Ka>0 zqJIKY0OPE?tthfdHN`t!h8D}iK+QuSFFFpxF$^8kJ|#w~ul@4H-+e_5NCl<<3O)VyMMI)f7Oy#b5mPUu6-AdK|H zdR_y+ZuR0WuvyUWKKFo!b5ScS`QD6glZqW^p!+18I9#&Nu9uEjLE+wF~=z=VFLz zBUac-hC7aQ9Hp_$EDMWiSeBsXf9B-A#yOY@A%%lFN2nR8H3Ng#XI1?iyP}+P;hoYk z2gLeMhy*Banl7mPvgHU_DP*9|cV=4i#bxQ@@*-T`?Lfono8G>KW!(n{I_ra%-CCc; zX_hFDCW)dg+=K|k(5rDW?+aEn<^qr>C)R>OK2MWcct4q75dNNr!zV#GVa|l$=?msG zs1c#gOb*6-z2EOj;%x*4t~~=!792av(>Yj^;M-ZDdDP9YccQ=y_d)o8!PAkD zRMqhHs1Jc^f{khfSgG%#s?yulYLpUWt7SFl1N&J#KE*H@sj)i; zwMkk#2wGLP4TgdcCQTVzy99;MSs)$*Zx_Qx(+B2eJ84DOE*L}zOe=>I)1jSN6(Ed` zlbN7;jYXk`fLFsqwU$9gb^^YJ91{3T2tPr(!67hI`CKQ60zYM`1b5A6_X_)VKk-cr zkBl1GWra}Ds_Seoo~)VDUtNKU;kkN!Y3-edxQbpdWAMw!K=-tX8hRQoGpVv3F&;&G zVG$jIItb5Gzjd*whC4c&F={U=GyYf#l~W(lL_qyzlIB88c`-1<(<6v;;+_ z9(NqSpvDImw_T3v|2?#Kq2*5&EqDalZw9!K~cR^OWD%r3@sQK;iE38{wdAWe@( z+}P~U@nkNzj^=6|p`*6LNWeGMr@xrAQa1n?APY<#8+>95km9-j7Hciaw~(>872zC& ze`$dY!%)~tq3v`8Z`@$6nWz;q+_^j~s}v+Y1U9r>)N>x=8oGtlo5p$#+#Yb?s_ad2 z%*L-GyT#ik-0v$=&&xo=x4LrgIHi$RS62foW%i4cu?IHSz zonA;&PfWgI{Q4&*h}Qns!Xg;i=1|~4* zBW$Om1KwY$ z3p&TA0#hJS`p(C%FTdzmU`9F`n~pxr{Fsy(6rFQGlhZk1ej=@PwV}!CW<@H3CYKQ8 zDAL56TlJ|1Q5u9FzZEP@eTe-Z{1x$r3>~Ba0_{#>yT`I!?+g`9N6dUvyBtWRgi83d0?nC35|FBv86x9LZ4rKVZ{;^^O z1k?H+)f6>HS{w%uEs89ZEdX3=P|L8$4Y)hVM2)+sTO%jGOYle+aNyofkYsx2I0Vzv z8rg$1s583mUQi&}sdEy-G{x(IxG7k@)GGf+0-X$E1;#gM3tVc)eDTUkFg@6{ASv?H&{aB2;HPv#9S?>nGQ?n zTO4-4Vav?H2V)9TqS|X%d&9^uDyLqz`kighLpECdu_Y)VF#(aBC+7|)XY?>p;Uhp+ ze}d=6&RX@tSrb#Xz=>LSe-%b%w5!)jK{-E zSYf4^^v#pLV%?4hwzuPNm;xBd`>6s!YAH3^NB&jKBXFoO`kW4Ow?4c{SL9@XxHXA? z>z~+)?+e4V-A}bYIPU%X4Z6pLA+WffSd4iBexsIh%$_w1Y)7|6#@}!DC z>7Uq$bl8bH5x+JDTgRF;91Gg6Sy>Hm+;h`$h#;A(QERj4OA~KbDb&H3>W9Q=B(!*1 z(XnEqdll*!4ve2@d9E?Wv^C9Y1n8K+URH1%WUheJW1D6HAtUy{jmg#OLXLnYZkhHV zS4R=?Gtj6X1l6s+ia-AU!K%hVK(aR#vy95j>r1}jhrkq?uKpD;TS)Br@v-lT8X`1m zQzQH)n%AFT$>Q%1d-&ZcDO;fM+M^e{Oftq*`zoOgY5%}rAX**~-342!crl3Uh9PJ@ z#G$EEfDk|qBM3kKII)Rn8T2R0paGF0T+K)^9R|bbiwMVWK)h!rx>Zsn69#LAsZR3{l@z6Nb6CilNySv z435!1aY4KpEAvy{3f^A0)OnbI$9;vimZbxnu}Zi@0!waWJ*#G>*a6fJ4EFXrOzZLJ ztg;Odi9~sTf$PWwbpk*z9!5k;SmZ{He;l#@8}Bv;reMHt2c+3p zoC5D{*ouyNwCF%Qh1l2eWqV4dZs-HAXd?PnJ1)Q~mGd!n9CYi(lN@5~Fom3knKC!z9h%;ApHoW>- zOX=T=!@SxJ^CNTd?4xFgW08AnVZtgmVzD9G`?Dh_rik_uo;OG@9^HAb!!A(yEFhx; zgYIJ;&Sv=Wl}St@Z3pT=epcZFdS1mcKha(n3w!sE zEFL8=M{F@1kG{(rK=jh7OM66m@zRW{aw72Fa_X4$yj+6O_Yb~T?cV6 zg{ts&PsXXsWTZ||iQ4>$xhC>SvoCo(t+y8kXP=>bTt3g@Hu2Ina5~8?yV>B+YCB=< zY&LD4LIRp>+rHJ|pzhkKM1k&Y!a?^&RQq>8>BMzP4m=Yi}gBW8| zU`XnBfkkDep7dCFE11zK4+h-UOX4-#;uTl7dxIJcZ(@D>{yUR?I^o{NRA34Wf9``I zwrC$gTBVZeYAtm^cyg=Ro#SV`Id25;6zdK(!XL41+R#jM{Lh5@9j!dpPSR>;ORJb6 zp*N(Wsa1@(asd3mMDBmk+^tTl#zJ=gSSB5;Zfb{09IYZ6z|ai7z-akf{qf~aR5AQH zb!=W&PG{OO1&}GhA+i)>0x&sh$06|}NNdQr_g_!rt4N?aZs?^kiZ)u%X;BD}|>;a0hM#%Y=6TgYp2$4^*#00*|#@SUg}f zkqroQk?4FjT37top#&!94p+vyr_V|+&1M#>Y&$w4!*582ACl%3rikNtLX&)5q8G+| zVBq*W$svvWd=GFS`oje0$3K>->jmUGg79N6zeG`NgHoU$jrDTi8K%X?zWNpz*@~wi z{fP_;F^^(`jw@VBjkd5I2P9mHkFI-Ele5|*95eKBE{`NGd2%=d!5Igo14Ds}QzZeS zf2{g~)8vv~9Ev~RO-4!aRTWaAgD^0AS~Qo3=x_-!=6P1P*Z`9>bG&Cak!oVqG0%}$ zyzN=}Ek3x7q^~KPq%N@qAf&>~n?e!^rvfXiy(K+Nn&U{>fMPdKiHBxM8tN#xv&(Q_ znu5Ddr6dm1>>|`xj-!kjvKm3z6v2-(wYTZD{da)%CS3^)~TAr z%8973SnO&M6duq8L0aqEbS+N7ddbRFKr6^(W}R_2qkxKJOS@zKIIp ztn%Qw%t)0Bn)TtmJSb`xy5k|N$WQRXyB~)?+HGpe7O+CJag3%Q^(a2z1+H4T@>DAl zOJ9rlY7E?+fWLjx@c9!K9hnMD0j+C`)wckSD{{q`NcbXRKlCR^U$SJ;v|SdaY=Ke(-jjaeD?GPv zj}l1uEr@vi3Fe;t@M+@?&ZKMs4Alo0pG<Gy7 zpKI%v3^~@0e<#L`K%RvexZJuBHx$792Gdzc(&$gnG=IU#>m9Od$`(L31dsF!ndzw9 zpp$*VQ!)d;Qh(ya?Ywl&{U@9U9oZE8L};R#ecT-BGaolA=G4My-3}|<5;N9J5iP4i zl>=OFGTvw(fH(p?Z5xA5M@g6w3Wo1 zuRZyT*Qyi}BEp$U_aF7AIl!(wj7I=Cu|W3>gBG3IX>XvLLQ)Pn_Y#cQ9T}^0CIykl z*f9cgtLg)voslfKVuw&;s`(V@a41B6msTuyM%^=a7(94M$K_}2)6QxZ%T!n`dq1TJ z;U^feX8sFjA3h)ErKrFm@7IfoP5O{Tbis%PC1kmAsxZ2sJVvfTA!>kGp{QxmA{oo` zB?*A{G6{h9N~u4A7?@{nHL~A$Y8-GpwHG+9qe@?OS<|Gi(n)=>tD76~oUdO4MPm%C zRT3-L%GHVQ3oz$@;a(T z*hvoNu;0hL1m`?;&>eSOfy6haUbx5JKiZjlO1L3Imi zbv9E_aoxR|A@ZoU!D^&0UmVO`W=uLPP<%&W{#sK5o4JnVdO2)crhqZ6#c;wjHdXZz zN(t~7?Wue99Yi_9_31l2 zf6JtlEvWwZb%JWO)IYuMUIb)$q`6*e^YE)_qys-uY*{ShwYpT3DmEt`i%jXS#sH{t zGd|@};)UY6lv6M;t6r0FiZuxG4y1-W{qJRDvT5+c+qqW^VbTwMsSu-ZMj_F?-g2R| ziFLig>0Ys9JXH%@-2`IkPjL3H$Lw{*H{h`}1qv3u3Gc+|8WrD!WdqiZ1@SJ(0>K7s z3WmSjerC1}yMJIwuX{4a)*ytq52sT04i)`PM{$5KZfE-pgT6BW<8~FoaBx#A3ihg| zPW9Fn;}}v$ugt9=70W~Q&U1v>#p~2d&XJS7%)L?DH1TDyK*&#U`m_l%H@_apaIgBV^9p;~Oy%dP)C1p1vcqQz|e8)C6(+g|sd~i_`mKE?dGq8emB;g18cd=i-5= zkaCvpZsXp2{95;t$X z#w(lYK?;e@f|`#ivb-e@3=cP$LNqh#7FHXgGoi@!)Nn_3#*Zb4EhN!*;tBUrvQa=w z)dCFP+8wR&6AXBH^^-GR4O6yYP*+cC)G3|{ndJ-^OH8YQT zYfM`+Bi$TBDi4mOd~qz`8>u}0tA&M@t&g$U{y-k5(4UV5TKoi$75X=)zJ+tJDJZ@o zf^oSik1>pBS+N|uu)>rA)V;6ipJ;y}d zI1x~5{u$pFGSUybM-8T5kd-!`9r6<$?_xoj z$uYZ`h(9J%Jrf^bR3Sf6bD(mA)-Z4uS6kut;FW606I0>*$&%5}F`3opRHZI;>awb@ zXejO@`r~J!X86fwbHyJ$5pF-++Jen?TDm2lFJXY0LUZ^XPSikj0W%+l;I<2oQ>KVs zsPtPgL()`RqvRM)s+f3Z8~Chd32EHEZQM@Q|CF2!H*P(b9bwKox2tUS)#^h`mQhR5 z>c(VlZ9|ubS>(mlO<{<2>LpQei=oa6eI4`*`3VO7?8I{6nC(-x(5?9P0DA1=4Dl08 zYQFj9`}c(2Y6|Kk0jWkRcIE-y-(@EY09oa}sZ&esAq~5^G0V*^Ks2nk3eHVJuZT%( z;ey1wA7aJ39F@xBsqko14>LjN7#C+(a6S7+1y5V|87JC2+?w*Y>20 z6S!CgDWP=5NHdSs!7A;8K>gUW5h!A@>MM!QlZ)a?q_-ZDtfw{4t+C1L=P;PqPCNoO zkZ2*t#g88WhON5~62>MB(J&sHN31IQ0g3V$IA6^#X@?>|VhK9+x`u+GFa3oFlcPr% zrBD*+qNZbGZ%6N&d8-;gg>DY$B ztY1Y&!`EEd;YK`}@Dn_6OYn#6BA9ec!EWJ~5i@=wgBenriwSKo;vhfA13lUp9=pUE zP9tF(cwpugs6Fs6$$cUlLQHh^VrNj>XZEJlyjb-=Ma z8hQSBkInrlfu+ycks9sxBG0U<4imO@Al}ZYyt9sg#H@P}XP%+J>PpwbWKrG#njoBcT<{^{J)snmO`VmiA< zi^vFznGWnsH5Y5lZQ$80x{cGL+HVl^CMElHDn8$*$wbMxt5kf&IBWfh-XX>^lE4JK z<){83>!^-@4=X_1O<%?wG826#qC%+_G19e8HAq~oF#In%D32{br-n+^ktrb45TAz2 zV_p*tc)O5#NoIiCpFBnAEi>bqa`Wg}Ie`ZEiOSVzr96Uq9c;d;Ro}Wq|BkPXuOTo6 ztl@nUYq(dhb1|-^*rcOdqKCT9fn+XE)W4A{sCb8x+CN;&QCyY0-xVXKtkjq$)i?6Q zWObBWd^SZW4f(XOmm6?Jx`T;*czaB)kO^^;mofW57c!Wc)Qegvpqyi>iSjW0Pb8$q zSP=ZBYY*rCFhSd;qPxWA z`ms!U0!?q}V~x?I9;VnzlxSJ>QJr!rQP~sYK;Z|dtmECv!{xCQW;ql4-b~$gLVXEyi|>np4nO9AerN{!B4bx=$Wxz37hT?kiBTxNj4`p2qRHS`@1CfVJjo+;cGF2Q@~>@kH#B!QSG zW+5Z`DzX=Hx(!Mn_fE>Y@+fU0;cxSJI55=Ph%-W34m1TImQE-eXbA2q6PHp@yJ?sYC(r}GG8A;j_ri&t8?WcI^vk}xyG7~nMe|Z zU%O?`gvS=l2Yo>K@Sj*d`V%}qVB!Z0BWJ=P6TLEn6v=BpspgwB6Zch6%hhcjvMo)bh9W5!3iO$7j zmI3Y6iVS|Mf9zs`jJ<{AJ)axM2C$cC<8QXD&TAa8*@S?57UQmjXcRB=bs>xvJ5bP( zZ$Y$wbO{9fP%X4$EueqhLWwPm!V90_5*{_%!)3fc(yO}aHK0)5pe@^r+A-ZX>Nz}t zwQ>H47*3x*oN`l3F(bl~&~pIdDhoi;?Eykw@R%upM6D=Se!v0ZHXrjp>pqi=N z+CE6O9v9b<`VgJD$S`zJ;g(p45#cqKj8X_w2p4k(+~CZ95}BcH<|EAi9Yo!uEIPcs zJP?N+++qWE40+I>;Q7D*dd=6i**s+nnBns{Rbx1`{xAh5*vU+gohbv($iXVdfiuYn z37~bD0_OJ&G!@Pz9;;AFbmCFWfd2vD#@`%~J5kZYPMsxfJ6oX9s_fk|4B$1Qavj^| zJG<>jON@7G5)BJRv2_59bgfUJ@*#c+OAl9dAjdV6R?mI1-Z|{;=fg1YZ-Qp%rW!o9 z(81Gvtb)+Nrc)k-djciqbI#zUVU9#|GthIkcke(4PVR%04AGL)3D0;^hkXNUEqqiT z$0Qk(ezqh(1r!U;J5A7c=ADmuuMi)br7&}D3C0V#tmX{EAD18RxrwH z_pNXc5dLg_5JAWQT$I#}qIw+6C2DFu3x#$o?EP zFFrLL9^_i^6fLwcpR@f5TNokM3*+cYOtU$x+4S&JBGXHu0X2upMw7-Npq4h0l!rXr zhMY7AAKRn8#DyNK_j_b8F{`N8N--|B8l|`m*oFK=f%-?Jg3QdLrflu(bJd0~ zav6`pby&Q#Ag@9*MjI~RUy-}s1W53}cgIb;;bz>2O?|uj-A%ULEv3S0DWMLZy5%Z*6fkBTiu?#?qM^M* zp2T$(WDk!enl=?qBJ(xUptBFo_&Va*)$(FFF(XvL!zPk;DCJ_UC{vHqqpKbIR)<{e z@G^x1AMMh<2Ln*Ug*C3@9P?vCBW%GuXY3Rv-)S>RQ!QM9#`GuhtRbjh#M@SM9rkhN zd1F4)wlZ$Xcfl6{oe774wEi2Wwa#ArUqevv4h0tT1!nI99`|50&2R^-R()V2UR_rX(ykyaMwV@8ZiNh+v}`f`NiH_=u) zMdbLgIN1W|^bY^}B&&{tB|zZ-ip`0S5f>Rtfmez#Jg06M!)r8Qw7~ZHF1$1o>TBTz zA~`0_f9e;s!h9NU!iu}nsUsqeV!`~1On^Z+wMy1=EW$}$0ai5VcG=pYor>1ykq@;A1&H>U>QzP11c}2rDedyI;68)TE!W zq563y$pX6zajNnydxvwi3xI174aP*npWq+IT-eUjtyHuw zxP#mLV<+*%u-0xyt=i}sI+YmFWJLXh3C|Qv9lgg6j)ki` zW!T%Zutaq~mxEMT`BdqOSz2jbKSn*bdb$Zo-y3xlKlb{wEtrFmdzHfE&k3$u^~b$0 zJZtZiY6Op>xdFiRnZ|o)$$W64S;WF8)cu-I>Zy6^ z89igMa5GPOsk>!7ZWE~`JJtZGLd7rApF1;3zqRUi68ug7I8N`9hFv_LuU2x81}zN%XF&fPjQ^_tLwTp7mWzEJ#HV5iITavAkK zQTjZRfVp_mQiC`Z!=R3LB9;S*GxYnZPV?}cP&>7qe;sHiJdq%)q{T<@KgEv?Rj>t+O@n`p z;Buodc5ahl#4_uuzetN$u|jSEkMuH?|}0KJ-oF7+&u}7>C>rzUPeX3n{MJf8{q>240vo`6>EN`I=1i5`_E1;XDge(Qo0JxosF~!=? zL?s%*0ccEq!e+)vGYkWZw}P=a3!Eo<%vg5D#Ps@Z3%Y#}1W4QOB1E;7feRvH?^ zw5dt4RUVF-hN`6%=#mFfIgJXdBF*@hZk}_+htCW%U530BRUR3u7){51Xt1V5P+6aG!o#SAVQ& z-WDRthk2|fgr4cp9*b3q!{Nu;vTQ+ps&^n?1*VU=Gem{>Lla^8Z*#3ov4RFK#-2S9FNV5>oC6&kh=PS zgmCOL*mYXXY%y*Hj%T31@R1<^Uyagd&YcE77rQ8&aW*IcPXu&z^0ZNBVVIx~m%r1> z2vacF9Jw)?hjxYdA``!Zw(E>qeO*wmUMDdHO#M!1S1tY$#OMpZf84;Omt_<<@e4Ba z1PBNuA=ew8{$pC_@s8|VZsIUJ{S#kB=ioP$zv5FKjMokM*fkf1+l&me+n5@VmvzDr zC$4g%ey^zowGeq!(HqgXv2LQ3pP-7=dVC~m4TPBm=q6@EmP?HqL$PlgW$9Qh0)CEtN#Y*U}EGS*N7qaWG)jxsnAPK|LQZ=zG>@MY-^hQJz}YEVM>5mot1+Q{s87x z_wlnzz+YgQpPKX}csLr=j--_d8gio$_AOHJ%(n1s;D_kz4qd17r8Vym|kp>|8 zC%)ELIXsZ3hK-|O8+}BQXsj9$PhK19CCmo8aT=dcHMtJnK|MMDE*9_`@IUQ!vy0dk z)FPP;7g1BI#rM&oTE)Nqu!II z-Pp|1u*jOGM}o;B$(OS;%qlV;R;K(6)Z_jBE)X6{D!|JPeNz z?Y#c&YI<0yvXIarF4|4ZNR7tzqznaQdAK~t6S+mvHq$4_!$>B>q%*O4-bVYjqnOdR zN|_Wbj)q{t(hm?5=G)9Nplo$TNj0QPrnebr>^jYk!HP z19A0?h_Dn_BhX=D-h9L(m!}(>oN)TP92v&_=s@JWzFsN{Us^AvR##7uxC}Q-1^?RP z1ofvWd!*D~1>_P^h#$+Awg3u!mw&y8pH=t(bvHVT*PO3yzjOE1+oYmrn!=MXx%v~4 zdWTfdHfgs3=MUJ8XDm85!!MJa$I2rVQS_aTGYm_78T5#d)P{CBum2ageQj0lS__2O=7KJ?$lJ_dz1G}sbV#0{dGMI zC?;s-Ef4i-bt~p@VZDr?m3B92JKb<;(M`FrRYtt^4$m`VICiG*DGvr^w6zvkRG$%I z3WvhZOGI6>9zOz$bAW2t00hb##h%UMwVvq8Go^*oAVpdre!#@>0W&(Mb^1G_&tJt~ z@M9zHY#{;<_2u3nHvSZS)KNiArK!LbW@cUphJz%R&m~Pdo|r_vZ=s655>nm-L&`2| z%@*ha(C2Ha(s(OI0u{y)XMXGkXA9WAOJBS-vHGT}1Wj}xKSu|m0`$AX^se(|4cvVO zfvUH(x6DTV7JW{ApkO2(Hnq86bu^3-7*BDz@?)i-l~ODFs|TiHc^o_k7=a%!5jC71$8^LLQAawH(jjW9q_nZ^MCLvgi) z`D4o;#L<^~SFqMlUh8!>U8nGS)zH~WqO~DEtMEZD&dy<%zbP2-!+>DY(lRv!$c zRab()la;CkO_D)oqEdLeSEtiw7?q0SS$)}oH$#|dPksH$(^ z*FfC~RDX2QS+=^Pn6)jrqUwZ*QE}#4(w-W93EQT~G3si!;+sW4sajUVsd*~aJ3!n? z)FaiDC}QgxK&qcfiw@i$WKlpEJ1YrVd`@aNW>}6;-mpLg zZQuxS9#>1MsbGor!(r!t=jc~Arm{YP1h|U~vr0I_4cCI3LUP*c+}=hsBZG}Dz+osX z;}huvho^!E3;&xr``2=LVG+6x@5OxaJvda+(Jt=vA%h!Y;>8hL72R2Ip*i;4XbL?A z#3Z)F&-;3VL`3lQ2L0Tb0*%R(E_6jOhF)w6=IVfRRLV&r_VlgVURZk?r+^>3JJ^Ds zr9dX+aijY^-fZF-sBeqX?Vm?Z5q5Cg16ciIC8F{2yLID%fWdTV!E_2uYH0JLCm=2X zs2nn7@G)tj4wQxD0vsBu?JMe`=Wonid4m>;HOFRY&oDI93K6`XEA@0*b&=! zzP_kdn@P``g6<(dkriPRbN6Ty3Z_%ptUd?~VlaYEARf!$B9j?7JhY_V#;@&j54NB_6uxt&KeJKVty6tm4dSmpZ>R9s5!k)T0S)518C2pkJvkn# z-2{16)N#V(PJtAV_`Vn=C#loQl*k9kW3&3Jv{=VNZoqe=FPaY)A`d$c*lwlSb*rnhX(%`EQ;F<(M1w%KEj;Jb&?uGb_-Oh52q^* z*^dz!3O$4mPE}U|Da~-y8A6+C5W90|f2&3-0|`_fikoe0xMy_Cu#9)_4DgZSr>0-x zHPYu$?3H-N3Gt1ktiEKD!*xo1H@C)$it)ip(G%5?oa9w#3E-ny<|i`t#r7P~SF4q4 z;O$Bty)Hprk>WP$!X(HwmEK3C@_<}hNP;>w(&Nntr*JLd2a{IOCG{TpLu!Q`u3XJ*=~$f0r4^kTAl?4ApSZF^WCWpcZM`unl;6EXfqs=4B`s9PBQ<;&;QOB@z z#4gz6e~;M&h-1+n+kX^l6dZ1hv4|MRW>To0GRxpM+rd1++6;2(442_x7Uh`gMYjvc zzeDgWE3yfykhi{iQE5EArvpOCVd940OS5M7r+< z85;mo9-D4wjF^68DP8QurU~9kI)*(eOo`46FH=5w9B;W$#1$bz7KeitM(UD`!Wv zr(fAVKWaAC_&PI;7JtkZ2V%799^qf7J9-GH?F7eqAlIW-LKF#&VnW1pN++x7w9cs2 z)`Hj7XjmSO_HNiu!LulAj4@t#0tE9F38s3@?o=C5=pTYHbb{&CAo#cbakKVQ{JX|* zV`4<@D)`sonuP6VMIGBuXJ#2}Kb86DRBa0rN%X$gX!Fq5Xq~u%L4zT$CYc?_34OEC zL_!}sjwkf35z!QUNg=}IL!;n-NaaSm4j6b|9?&L!Lni=)Xh7E#dOzS$8@mq1?1uaV zjrV+a-UXdVwqXk0#>jICiNa4be{Y$Ie5=_%6n}qI@t|jx%t{5OfDzD>Sy5=9ya`^r z!WeR;5INY29a+Q!MmMhW@Cj8ACWs5hB)_?tXbWmGew`2zfYm>>$G*0XeU~za_T9a( z==3MJb-yQbYi8gXV|mhdfu8M~$@%nhs9y`Cxm{15McObGEZAq0uMR(FK5ryxa30vz zUDe>m&}V%zYJZ?!do<<<1G>@5S>EsIs3t0m zVJr{DB@4B@z>_h_><~(AG<^I83qTuFNRoMC&o@|AzALA5bGp^)=}OGstBrZ&tnwJ% z2KCd0*QuXGrcT02oI9f2JZg7vp8iCd5lq!0b%ww}DA_1gmN5L-+$pvIMt>H_h8tmV zlm|sw6+tqGrMGpPDic^`pybv`)Idl2r}j$5GpstN?9?zfQQpg#sQ=9*&D7Gfn##_+ ze&CPRpZvlDQ&WK{bgL{dMvrx15|pf$lo^SExO#dV?&{`lHAqBN9hgLSD|}H^Twa{U zc)fPyWzY$9j{||^$t}@F2Bji>bC4EQPvg&(9}?y%d@$5H@X5Y+oEy<}a;8I5Cak#Q06iB3O2GK5Ta)k&qnZ?J@<5jdg!ZRXUOw>dJqBy}rPc&M$>`6$7c$ zUvePR>_F16y)pDH{r_nwYA$9`F?Y3*=eNC%A6b^mlIE7B^o_ftZlb!pYd<%7dZ?F-45*lGoV;8@v;5 z3M>1ohgfBH1FDS9v@ruXQ_w`O*P^mJj>*Y7_JW3~yqs3uk=Ov^Ic!kXZO8T^ohg{A zx3KEiRzz$@23viWkLi>Er^;K=Ucm^Q%A;I*_XeIs+0RFAQ0C@}>-RaW z`8k%g(W0hc7F_rks8t@(+NJoMUepROcaqg(8sj*1AYLwqB(|G+1lBK0PlKsVqj@!l z%};Rc2@~#r>q{+=`cZ*s9>c{-+dS@3A(g5&na!pG%1`j_Z?+gV_k)vCwgAHF8ag&_ zU|3{eH3sC4PqlLX=o*&;g>pk!YzrB=<+xBDI)}1v3d!O5=Pel?rlO$pmc)qZvyD$p z4<^}NPtM!FI&O-LhsHmVb3|Hpnd{KfVjiPaYKLp_+ufq6;ntcSVQv?1lw^rx1{qnb&}-# zM29J)=6mnVdBu0O+zR^ZRFDSU_2(ks$3FXE3#v&#+#2=6q`xdkI}mHpH3gHBRoD((|Ft zXH_n)nuSMz5A`QFV9wNKPaJ)6$`&|tyzx+BfMmHrw+K49%|c%C9B1-ej7ve6+r1YD zuVgNm_MC2+$127cmWM{ah;r#T!QrVb>SdW07vL%xv!|Ji+0#<$c;F7#{|6X{yxuzQ z;EPV(=E4Q3z!X4pZ1}kSh{lkk45?;IOrnO7x~}E*;|Zz26hN8|R$2tds|nGd4MGeN zo&-1GPshNWWU1vgFn1MkU7vHV%&;`!csJbgMVPUzk_7oV*3-qiAy?nL-jpcN9WEZKUDVS#Pnv&5Z(kwZN=5 zxy9Wd?Im#a;Cn(OfEA%oh039kIB;jOhe#-X>?z$AETL$@)n;7l0kj2ml5Ai(OsTx& zXnh){piUML$G7G$&cE_6;Rq*Fyd2fZQEqQJ0)aB0ImX$+d50ps2})BnmK4to`Z9w zH?74T2g_n-xwShr{RhC6kbEQmTVz3*?qeh1Xj z`eLq*Zgq(CSFIVwV3Pf9;_)wo<)`t(zJ_w@_=$pCL0l1_lb}U;siS49Y&3gjBeVmt zTmBRbifc`v9Iw2iZJrP6;HYNHFr3)-v>^YiYm~0Q6#TQUw@dqHT@hGuS8jA0&Iv3} zY9vYOpD#2?wHqTjf6d=}>dNmF-%ACi(5=n`BPVEM{$yd2o%xn!-5Fj_%a`(aLovyq zp4vgl(+Agq5=r!l5bhFF+q(mtR30vDz&pT@G#IQ~93Fnj!TO0HX;{c~WT)ji(j6Iw z^R?$WVkZQRnsyfnp$UB%Zxass4tA;XB@|dJfEs8k{`ibq%-8gI+>8bUgwG7 ztR3&Iy)Zv5kLQPyLpX5$#~T-)!HWCS`3Z6x9=mwE+mB8K=^lW^g;9X7L%9LIQ|H3R{TUt#MUOi_tp>}d@db5yW`WxE8l+d z(3Xj*z!c0lf5dSPMC72jU7S%!7Hceg1vQ{=_Y1E<4aE~cg1x-kp~?B!ajtnWYT|)M z9yx!3d1qW_WuBNRO`N`L#`IacOmCe%b9!=Ji8~P1yX~gj-0#Q3cGOd(5&0ew;bS%l zL_GKeA}ie1EP-wN=(N!FNv%DK{P^+4kb8i%mUc^8M@J1 zbc~HRazsqV%y)@&cPMx;2^3bu?W&nJtya|U-Dw>9Zui&&Q-LXT`>z+j`hPzkFn+!L zQh_OeUQ}f&`{UzJp7#(*64}XifzUJJ@Z29}w1ej1-)?!h?M-&hwaw9Kb8R!{Ok0|s zAOqbaoLmP!iNkglhoy+Tj7?PyB;0Be3&$?TWwZ-sia`wE+&VDHj@{sk*(^Om?Gq=b zAslM->twXIio4Zv;gC8o$$odR14h=Z98J!3`*wt!_uK2vp4|_@`KDm|t^YpBbq7JttGuFbbPQTqM>bMR} zk|<7uRc#Y{ceP{3%(hXCWL080a97n^Ym`oPpfNKI1Sj%*15T_1lR#jtLqLn~*tGPF zfKx>5J8b{C?AagRa_?;IJ=FsJLQE20)S*ab2r@5l*Qu98%2_w6rlUuk^c51hNAi_P zuiW$L1^c9eOG+1?_56SXK@{(>s;nzE^Z{OTijJAA9Mze`cCC1P-su zeo_af9u5@dy}==zh56T?;J0V*{Js4$K`NNsG;_WC|Gx2_-R8EBJJp@BXYk~YUxQO) z+^khEoHgp?lz-c2^$vHvclqYH5VZfT2mEyWUpL-U%?1DAY`6_(Sbu_3@~8dZzGv;5 z3NHV{FJE}?$+i@PvW`gI9rBn@ZD-&nly?^rtm)a9k`!M$)bggTuihu|v^j^}C)gGn?MKQ_C%Ep)3LHJ|7|_W;vV@BZdgvv)TxnGu8+ znnDfr#Uuc=nLyQ2A&(PFcG;Rj59f&8bWH?%XM+yM+62PjmqMD` z2*XcgEx-U|O^i+%diW5kpY@2-fO#MsRJc8}>6MXsY{r`SZ3d>5oS^L=2puUD)-Z;%S8q<>41P@q; z>9A8j5Cfs$q=?`HhrZzlkcKBpZ8^o=q>@^mP97nhKoP|Su0~@iF?H`?1FM9rZhWQ( zYx6eXhht>rt$29lGuup3!>l!J+4c;w(SJto@P?V*jW~1rkJYRML}#MV$L0GdAsgAF zqLnS98@XxE#!{-hfK>Nwopt)sm+pHM7K}Pr)};eSb^T*U6A>re4v=^b z2m)8XG307k9L)Gl4N7&>ffMbv17klki90UAUnE#pmLDG9FgI1uA~VZY{s0iRK7RTlhvK+2)DPog6&F znGAF2=J|?P#vJ(An5D%WwH@_XrsW?^@6i&(}S^X z>U1nRG&A0&X)2-(ZjktGdN8)l>o~4xk&Gct_=&dl*rn)h^&9*iQ&9)Twn@beN&u31 z@#dAvcII)xv50whgOpu8(ZNH~K^+*|!H+>RnT!S<0HQz9`@^^_A?K0pL5K)x28r|l zW7}NDHuZBd4dH2EJN*gH{^_c#Ui$r)5P23Ay47`h!opdE;V1aPr$=wL>yrniY{7K< zKD*_|>i)K1yKN-Q_dqa^=V8tTKhfa~Dy@-$b|`gV*e3Al7hZQ8X! z>F;l~U)wT7M4kDquQncVA;_~805PXdm9|HLHx65R<+8`NrR>#-Xf9%tJhcM+7cBD= z>~_(~|MUIZaRlD#o;?Py_u<|aTw@Tu&2tw=#gDzJv7n-iRaT#Xde}tLp zz}PllVVm1y=n&A~;;ix$Z2ZRAo#)M5jPuc5&mHuYV^?SaUBv|hVq7rkqC4K6yglHj zF8S!jnMbThsRbH^n6_jT6uU!O6#hXJ08SJZW5?E?;Q6lnel2G%!h>bcAG_wThEp-D z|CM3A$)WQTO!)AgzpgysxK!}R$zN@0d!!>}c-2vAKy2zI5)j}AoHy(2UH4k7eMnog zcg?E|X~ufd`U5|~nK%CA%jpxh!M5=1r@KCSDJ;+6mE(bgx)PIC9u&ogC3xK&D5InC zq?GzMIsk}sD49%9fu!S_>d<(S%XYe;;ovX+Y-i1HjgF_4KW;U3I;#CWA~Qk%Gk_WZF(@a&C4-aYDP7=es$BC8~>xcQ7YKtz2|n= z@y;#KrWcDS%GyXcK7RS)@4lioO!?Bn>Vy*2H_(x18&1`>GL*}R0zbj%hh7;z{oOrM z!JkGSHuR)5N9oP}6Z|DykF%uHVDN2p%z<+ln6p9j9NvGdZtA3#_#)?~f|p+F2P4%{ z8joX$g5Krzsi{E+D@(-kpr4&sE*yh^(?$2+K4zOq84SN?mH52Xr|9!DJ4n(afSCl^OtIpoj<4l`{OIF!pIE_JN#YXot(Ii5PHVokmP|1rZVz3iV8ExX8~BOq27XTOz%9)U=PN$86!xX z2Zau?1C?#l@t_A|+jLZG@NSXTYLln@_)$LwW7~A3_3^Y0um`Y1N9tZRo-r8PrlWsD zEE$j?qm5a#ZI0$T;K!2M!q_&QlGG3~=tqBo&R6c5^0(XfP6a0&_>FH&KkYcIznv+N z;|2<0c-YfSXb_lCn|+vQ9(hZ&&B=vi=v?(^z@4#;#UL<#6*Rz?gp^L-CeJ1zaT2fB z;dV-*Z_6L%;IT5QO07BS8<7)Bcnt^FYW35?XrKm>&4N=ukVWPewD1eeI3^-h-5?;& zOktK^JbBQEWcmxZ7Jh3}&mu)^{Medp%Ao>phB@-Q)Vc0lCkXt);%B0y4OqY3H zGKO7)wsH?OPRVg8)l%KAQTHPQ@%WUY)c{?)UB~)k=3aTp7I*%3R!WUPp9v6SqKU55 z&DOh)R^y#s-cb1_iF-GuAt;@C+nmf z?`q6!T*JNB%{cCWqh?XZ(>zFbM0p(S46qAlmLPLdq^x%$!U%|0bCMMS&okHAHmDrl zhraYfbyx$qTvrwKC5K6cHFfnP^p#ZN%eoWyQk( za?F2DJ^tn+Hv(Nxlf~w6_F!zA?oEL+twdd8?*c|N*&e96M&L(xOozDT4BKvxXg6O7u=N)y;8G!%w z{KIa$;leXhK8tkj^58CqUWIo&M1Jq2QQ(y$bu3y?#}Va*n79->`IuC2>lvk6!t1Al z+};HO4~A2y+Z;vI+N4dVQOW4>0PM$|^Dg}3zeea`l>r(qX}|&C+BWOJ zm^Q=BiR)n6_9A5e7r#*aL^jhzg!eHdU3~Yl+lVQsRsnHzYN$?sYbJwaQV$Eb8@|z6 zA`j6g+OV@2P!j6XLHM}++{RXz2Q$0XHPWWjraTx9UmN_mI1~ds(B_pSFF(O0{r7nK zjtuNz6VA!b`o?vzVyl^05Xdy6m)av}-|py{tLNn84|>Y-LwzI zS=uCT3`DbBXPnL2ItMJRrsY9-{bUD4vfJ{kPH9firphy{G8CPJ?aR}hxNL%ImykX9 zi7bwY_Yh5TYrVkq*!y$5`q9Jq*kvN;qykf*46dTpfEcX|n1#2r-45+D*>$nRr`_Xz zaBq&g6$Li%c5WjN#Qu2^xGYBTF0LN`t3gXX zo@}mfBfq&iU6!01nrH`LJGJU-Pu$osFQqmS5GNR9hT-TnNt;$iuPJqb!sW5Wh54=y zDES#6;x?YxsSlye!wAn$ zWCehS=I%>BdS$ykcS@-a-EH_y{I7BsL0(;O!cp&^xK}FT*CSGXA~`FgrGRX1>1yQd z$;GT9;64*Tp?=9~(Vo3*GAXn)vl#?1Ub|2|ASUaeJbrp*+>LGbU8pjM*GL7~7`K z1V9AQrrsy>C*P|R$4``R0TRNbv*}hl5v2k3 zV9?!)Q@;51Us&FRgwMG)WJ^OF2~frRt;Ht9pA{BU8dSax#LA6n&_fy z;M6_I&r_s*T((*1u)~io`oj(vwipwiUgMV+;I5G=MG9)Uv&> zmim+IB@Q0H)IS!!TeS)1bzn@J;dD~tEfP46xoy)}_3E{0U~HR=h%`#J!@*K-*{};f z*^$&(*S7XyMgi9|r(Cd6#4rO2FHx7=j86P^QrBXHYcgMld() z1BY&tkARg2;RQWjBTUf4t3%w^aF|1ZLU{w({ICb|0vX=2&@WiHp?lTsFtm+Uha}SG z)KU73Ue{qXmFt1oipIGeNch9kUh+<%H5!|OB~%?^TC5M`3>+lHccYZa!nX0s`@fs4 z*Djg>{S_}L&4gL7aq%8 z9}`omA|S3?b8NL4OKi7$08}*-K28N=tq=b6?#JPeb^{e6*3j<}l?UQNGY-5Ugu~RU zV!Dtgj9#nm;)7FmI^fHvA$P49pZ3N6?2T;uBX`;8^cP^eJ7&|zK3+5e52kt}vq^BC zdGxr(13%hA>qlhba3_55d<0EL@fZN3p4)p(HR1|97>)%~6X^x+f?Mo5QxRF_pcyPi znue-G4bBu~ku?RbpR>?#IF2(OT}`b6pTxu6q=3#^Qu*kVF1gz7epHy}#h%~TdYBTU>C>A|qgBf(ykcgbTnqG2p? zy;HyGhj(6l=hCm}iIg$)s1)@^N>Kh7!%rkb1K=?PKtUNa|t|=~i@-Ad;O=Q!pyC zdYc@hi5l0g3|K<7tpMu^r=>h5Qd1*MTlCcUq!?u|*tKy~!z@k;5o81EHGW>NCVp^~2;e$52SmwvZ^+#dMjgT(PxCiM{2gbH}E{VY{EzzbC z7*=wS4$PAsp3s1a9j1@f=|%Y#2@Y``c@<4ZIya*_vbrO?&G{L=R1eN>=qTcs2H4~ zrr=Xsti!h2YAs4k4XB9U! zY}F)ETYLMg#>kBjxgxm5x&~!F$D3U2W=< z_Ym#BG^EnvBPltxN61N9Agh1ucEL%uJfw~io~r{-LTcl|`2bZvua=y)eIpUN)qB#W zlinVTZIcEG^=HES!UEF!{C>G{O?u1vcXgZ6T?~e8K8u0r8?)uHI^s$?@L+74+M{`Ab(Vc zqDK1={r_3<^@b1nPfI1hr&g0qTE*m>-O!@*ET5b_c>sX zHb(<<{fVrHa53I6_l%p5xN$cK@Pgo32gbH}0y&kjI%rettx@WU&9l z6uw=Jvj=0_e2;DFL`d;AWzDTNAzsKX6-{BtDNzm8ID7E6{f&T%PZNQM2r!6stNF5? z934Cu+a^^spAG^KIbjM2_p9Az42Es4VROu7sdf=f+2J7QLA06B)^7D~ZQ90Q*yis@ zChdxeLPMPY+{9W&W~*`F+ry;IIxq&$%1IClWu%XX_lnlAYE-dGP5YcO4z2P#R@p%A z6)bB<-j8%HpT=pWhET=^X-o;G_S3Q2=ru&Bv~HwEZOo_C{*;I6&yjmo@}N`Ap<}T; zs4Ew?=T#@h=~6-qmk&4o<1rxL6q)8aFot|`OYc3BJ5IZ*C5psIhAjqy(-j|J#&8|2 zJ$T#xP*BSp%N@Mo@@NYi2AtQIHmi4XkkGK2>R%fKb7Z`-w%fr(a|q#U3U?#2y47kS zmFw1`W?*cajcjvhb(?loN}(Ty;Vi0I^ zKa!T8NKHV*jDa!Z)y6HgR2bjceF<2XET1JPJyb8)0NZVA?O?Av*Kd4p z4b9X47E-u=Js8{OABpU$>Ne%F$q}jG=*Rv%zVoOSy>D<_;i;IwawEes7Qld^c(=%P z<5e@=ga>IpL~z%k*HPGxTZJsbYK59-_%4K)egP zxxPH%sz?5_e1aR$rT|qMKHk0uZ`(f`BT?hY#_{$?teOA9*@t5Yra<|v3AQDNFkFUU z1|RRqRIoK|PY$zqPxjK2IOw*;bZ;z+%2CIIv279?HK7`N*&-3$w&dK`ZaU*8UOM*L z^k8h8oFtzj=E7B!Xh9TYe5CiQWRZe_rD(a#7JzgrvJC!Xn%dZ*T0>*grVdlDgw3pi^}9>2+5&-3Qk zH)L0JyywB#p-_qn#p7+7sDWU^=Hmwq>XvlX8iz6_r#y@<-UGGdRSbA{9eEXPM|`J6 zvN^{)5;7pB-U#g)$a^rhO_u~N-lmDDh{FAk1wp<{SsKUFgRyP$gbYE;Cl%DssowSzl}>+3_} z5g(dll|)W})x*OhSGOJP_So}Ci~&e2#hmDl7G4G00e2!b4xdB^|2qDrF_*rK)DYvY z8-L!9e-3t1TM;JSrUzr&Bob=t>Nd?oRl#$Adg4Gu>u73aL`~LGh zd-lk!^LBN0b#--jb@lQ^o-G#jmTTWQ`gdoK2j;RxJ`)yF!!I{K7bM>jL^_qc6&33Jq>N^c)pvHZ}jUaX4QSUtPY z?uHbrlF%6Deyb899J-(K(FY8hL~mkydJwDhdl@+oTt>ONw&`J zdmh?l=S?Vb>4!CFh}pr8$U~QI6ZA%4h6%Qey2!S-+ecrm{`uX*fw?JZ;z@f#W0>oZ zPnz-yOq)AyxAYq?FY$*XygE;2y_i3vW7#}YL!J{-avvgC*R|p_c16JnkO}A|qRqhr z1&75SIgE{StNPOJ>PZtDDjmu*eSx9!$<@y=$e4WEv}cZczb8^`>WkHVF6;ifun^=a zs{->z=IJmgPo8D0s8&K_)#?FKx4FnO*j6)>VZD1pY4;j5#wX7Zy2oUI4}97jS{k=M z_}oY^YtR^Eq+A!X82+pMAFU@S~^i2+WQmp9zg(me-+vE+}H!F=IJQycjSK zDmH0CV|8D>7clDuGt7{s%2=z`C5v`vQ~_~6vT;gZ+T9k#cGsVB zhv^|7K;!%7p2X~-T_BDFmv68fqAyi&Z4p6V+;7Cxgr>B`Dt)o)T#bb{cXUxi>da=> zjqi8c!lR~o=i0OSKI!v($3a{Pz2QFu`a;-m4YsOip>k|1-UfXKS(#Jpfi^dJW1F>~ zx^XeEuB{u&O=t}B1!79tw#Z6~^@pPpE|Pmqv@l>MG=>>OmTC-I_OgfHT4(%D=GzJgi~;G4*3`{mF<8xM6=KKjFi2GTRHM!du~&d3<#A z8Z%-82^@gM;s>34^{e?9vwlOwhhNv{BR?sD#b})fb8sgM{$eiAN4_f4EjpN?#P(Gs zA60IWS9wfmoYsVJT?!*GA>xH!t1o0?Lo&fZ8cgsE+4^$HOO3C{#7Qu)a-1ty$Y7c! ziv&Uj^+of;NA9!!WHg3(HM*wsx7gI4cKf#j7kU?w!wrL4#x4Kd`M!U89^L|dM^B%h z9GeamLtpZd6~d3)a2k!ckW;t0s`1Uzwg_Ec;j-P9`^yE;P}tMT;WVKkCdX5^F&~LA zG?z!FFYRuq7^@&K{mEzy^B!W}OCIW+&tAz#_Q>XImLJ_{n^Ly%_}9*V=!4&Sf5VUI zH-faI+xSL*C3}qztN_26mHEN4yg2N)e`FtYo8-Q|7OPdbOJ~_miL z^ky2aOGf}ujgPF5Y19=T=p0L1)@^az{h!O9IOC{ZI8-((m*irsbsiWUq?FJre)$zJ z<<>xxReSM=aNcY4r}GBx7$Up;1hCFnQn>8ZhtTJ0IEaSG(@Y z*LOl*y>-7l^|{tX&hl#L1>=*!;#vyEPwVZWHNLZQ{DHgsq?}M!O5d1bCn;j(WWB(8D3)wC=VhBD3DjN^S!YPtbWyyO-UvV3j^yXP)VoVti9=2vRpf ziVgI#bOZ6!e|r8k3+-`Es7ehQtIC?hOgjIG+n>DbLC@^2d1b1KC!sOS3#rx{c|N0V zCYd02L~6%#Z$Ej@b|&AGW~@bHnEO*TZU!*aI?N-~FN_?-Ts3OY7$z^AbmP4ZiMooc z_bpY*`plh+e2EOYxSJ~3Eiz=1jTp(K#+A;vZu zKTiZS9;ciqdjbHl=y#Htzz;Z*#Osp1TA~ciTU_*WZ6XeOs?zo-r;bxli%Hx!eG`(30bx z{O^Ha`}KP3dn_)wWFn@qD!HlZiX+ctGcI=StjilF z?u0ceS;yJYi)t%Ma&bt&ZRx8Od0*1zee~o#(*_OmRjEb8s(4IamGykLiCiDsCDdd! z?QTnO0CfzGPC{dt>j2KJE0|Z3aXxGvIA8PNTJ?JrVeCnYr-cowI=_j#W~lqt#2JML zx?9Nb4qJLP zA$^&-gs>CPFsa7z!H3;ToF~Ozsg##c{wu%e)^_A%W<3e?!Zwxx)a<%3fI)}h z>a8n5a`|)ytMG%Ij0fKo+7@92wuMN5)@<*zbFS>Y?0 zqDH5ItgYf9V_+I{V5AKb`dv9e-?#B{b>tOL1REe(QPJG3=bFg74pp1#bT7hSPaDcfl8Gr!pAs@D}QGDrWT$9uTO4Z=k0$y$p$ila!}anjr|L#&-8uw$tz1$tn%@6dDE6F1t+0o~1*0l0)Bu3Jg$|_71o?S`hPsMXUTAbLZu+$yVnhAHFT94u{4NLb~~{I8%e; zPwU_a-h`Is;loC(gQ+_5r8MwTyYjx&vF@yRHVO5pQC4L=1|`ka+;Rlh+=3($2cX?0 z(n@G5OIa!@WlG!Fyr4ZP7C!R*9YpBKMh|W#gk}xk&llm*(Otel?ZSMvPR+Q!K#)^@ zg&_Zjxv6C1i^*uPCGHy6(MFEby84j6y`?w$cRnoe*TQv>C#VDSCUbA7{z&XW{w?eX8x$*Y)-3}LIesr#(%|ZDOq!VIghnyTqacfp z?r>x&&v;PK<#h`|V{7aV;Ia=U?(2PD1cxvZz;6ME8YiDxX`J)8QLHhBKw$YtHs^uM zw>k0HJ+Lr7>CDUL_iyqN`Y$W{I%(!m1H1W4bmS%-+43)yvNgY2tLOVi|F)E-th4+h zlD={MlKZX5Ik-D=bptk@pWgG<%T9cLyOKMw*c1|Gi-d$?j(}2MBvH>G&v>#{Nu}fk zy9kpzBlALiQCr0e>{qg@7)?YoD55n{Z-^-Pyb}G>U9&#&&fMI^t3hL!Tr8X3YxArT zOWuBGWaj=1m`YsS^`3s~#I3twnMlH6_Aj{$2@_iYJqluZJFHmFiQQ;XyPyE(?`no+ z?U=q(IL0bf;M?>^g06>^+;$?Ip!p&b+_UlIi>po>we*I5rNqznpCk+061-^!VxvK`*1coh&@6*EmDq;G#NqY24v}5?y&He;)(mMJFVzdI zBs5k%?g%>ussnQ}o1KsRdZGNGqQQE?(CfDt^5G-9mfZPNsQ5^THKyLhW1RU|&4W4Zh?UxGrns7`g0oLWf}sN@ zG{oE!g7)sQ+eqASE=!gqZUU{V)w-tR#j*nT6zQF2iK$rNVIw zv!Kln=qq^$XPmgA)vOo7=loQWz3(iYy@U_%Cu0R7qq|ap89&goLIu4q6-AIAy|l>0ji)-xtY8W{#h}R9TbMox1#t zSq~00eg%7ZPN@2fYe6!r=5a`1br|;$LhW?L#g<;usB1BM*mTpEbo%rQD-RVd@N4UT z06R7x*3<08>OYf4jgNe;f)G>l^0J~$NP1m0sGLrO0aXBTU5dW6yU)N!#ek|m8I2)- zPL}7eb}_edr~}D!bP=1QRiSn<=Lx;ymxlr$*ID_EvHB($k49%f%yF`48AWF6a55XC zQeF$kF2YAX(nK^?%FZlhpqA2tMY^bvAYg#_Ke}N4&kCtQV}%?_QN4igAgW;BlTOLm z97R>$k4*KbA3JHX+?ZA48!Lm~xEAr50#9ZeA3A@DVSkz33(AuVrv?o$r_cD`UzS5= zXk0Gb+A_tBMXKmLd-OVzI~v(P*2xoJBU4)|)=0hm#E3B4DpMF`Syz9 z-`1cve6HbWz*LbQTM+6PSCO;GEut~Zmw;J01{9Ga%UHRjG!vgzk$gmNrQAo(d0|~e zYg*uXIQx^t?I$$Ew4f=snHaS&wo4jdT-4A#oBz!jugT$NPyaXeuKXIY8uY|x?{Y6i zS{=42L`iU(R$t(BNehOtQyK!vul2=K6U~p(e3c zBkF}I!7j&dbnPC;@o?PR(n6uO35`kfX-dbZi;h?H`uy&^{XSgk&{Fp3h*=YE_;C_2 zlc1~8S?ymev)ZPmY*_O(UGA&H1!u{@DA}b&qr5u$xvndmuusWtA)I~9Qu)XxJh5x( zPiG*Gc$4P`FLmSrdzH`(IGBOpFZ_z6hLQ-z=R1ZV!{VRCRcj!!)#xo^PXt6o=$owh zA&l(1MF%Xp=X|DpPD0-j8r6yNyJV&Z67OQ{Wei4WbM>#y(Z)R^q8CL^BRWsw`imH9 zbU7<}5=b>2R|vmS^|2@fx>VkHg~22?USbHqf`%`+HnH=~RCzuuX_p0H8Ve_og%`X; zBUi<2vcAwt`0?o^&{u{v*80qf;b-nh>Da~YfCUE8VdUp>hod#13^iyBa|SV~N@%oQ zSyevrEFH*aAy$ULeR-pWo~q}*Y$m9+`*&Z!?9{3C*6!O`8%Jv##-+c7Y?27Ls1s;2 z;qpV&yc4~NXbkfW*8DW7d61MgzuobOl)J^M=y+xBeOj!LSVY;E-bVShUlk?lSR2$; zhp1?*nkM=eSX057-Ag%xg#4d*-0m&>H&mh6*D&Jg&}Rbid7$&0&0(UfWJcfaa?U~@ zPdQ68FlTI`z}a>p)=)nwDHMD-d0`-!FQYqs7IJ^3;>#CP*x%e6T`{i}tntQXTJSW%) zzVl%Rfo%D>-Em{6N|%fx6?M4u$6olb%YiICd#YyznaSxr&)LD`W)54w&iR)0oL#!y ztOve0V}0G0<0Y#9UpUh=H+Df#XPi~jvlGc#u4lpIm_H+1>pyvKu;7`e20PLgXo9bJ z7!TsGVf8pH1E_YMpnpjpw4QMmIeuGY8TzalK;3<*VtCCC)cKXsk4I{;q4P)+E~MFN zdj;jXgvCB0#TKpJ{Mg=iV)Y^#s|rK^cs6JZua@Qogf)A5ZuCQ}N7~b;i75VSd@zOY`OIU{(X-7RxqE|a<+62P+`7N+$+?xtYidx?XB1am zo;@kjW)}VIr{|2BvCZ&O_RMyJ5AQOejlQ@q`Buy%)aCJ=g*gQ>JJ37m%pL#TZG%0G zd~)O=1$$y~F>^QG_SWtGgn=mUTLhCOIWdyRZZmt||6Fp;%NrZw^nQVXFxD^9-tn)S z=T1cOm$g6s>6XPN1=B)NxnuA^a|J0$^I}ZW2xL0VzJiBF6|6yTO}BvfSA%ch2h(8I zFgR>UGbA*IxeEHFV=AW#OkJGF9{yp_`ula~YD(_8hjJ4d!`!!*$}OI=NO!DQ<4xwW z35{V+#Pm$NSIBi}TguYROd6kK)}S%W7bDCde~}fU_hq)zHDK1DG0dI%1U`eT3LXVl zhI2k^&=}@{H8A1WtNrK{GomrfWli^TU82D9ls-5w^U%!wZo2a+m;vR}%ahEeG=>@U zIo40xmXb&tPJ5ENCp3l`j6kkS-ey=}Q}@0T=Jmc~kATUaC0I70G0b2o;`2umr~{Wl z@>esbMb2liCJBvUhWS}ucaBvy-^p&=;m#rMVdQbqzQ1W?Tl}!*Wh-%ET}no8_<=sg zw~3W5ix|%VGodleVBg|h4CCRLWyak1!9HAt$>k7iU_xVIC24LG8cG6YKMx&Q*S6Bfm0Ni-Am7DqYGwJucK1v)I{fRHDlzP-DFXPh99aUx zaXfoMPiFjSCO?9%A;Vp0MvKS2ZSKF$-*we#P@hv}v9Sh?VLriHo<{oEgor?JH1IF` zo%>vu9f+AklH1Vn`)3dCpN(!SxgO$-EI~*!Rqi0p96TztR?lB#Pb_)cIWwN?0?ec;xT&xL z3krV+mK&$?O=uKT(#E$4W>5gRm>!GPvld@)@$@(Efga_UHE0a;8hVU}3ucJjkXZwc z&B$GfTLJUfqT-1>v6`sod)>dN7v0BW0N&g_8+ga@Ii^U2(7BRJRSh1lh35`{SH*&diL_9&s z%|Fn*i&Z*8LSvXr9qZl`ixHYgl*iVzxN)9{C!sOSC&=eJMNIJt5J*09;bXu3uH`FW zCRQk+F-(7I@9-j~EZ86t_vEu~J!bfNi=x9O7C50X%nsK5wxT>u<;6inuHDqIBs7Nk zVO4obn?jZJrQLlXtENHb`jgQZ=0`O!p?mTn4rXZ{Y+>KGmNWo-9`S0h#he3 zz$qVKT&;qc&=}@oPNJyMc2 zx~9g4sa$#s9E+WazU!2mW)?ku#nn6KCU_Wviq$$9zBm$hWt+x6S3*-!^7IhY-Qcp; z7MF+Ec3yf=d|L$27fzO+foUio(gU*STaQJ^`oqf2;}HDln8Ursrl)c*-@|mRkU}u z4wth>s+xrr2C*6(!==;?UE>#d4;u$5?`~%q{ezgreAt;KoE&&a2DCsTl8u}Q zblZnPH1B8h-Hye4mFsbeuMFitz#VEIoJ3(EIz~}hwa|kQemJ4wf1<5s_w=Pb~mSah=95rYv>m532TT2rzZ~CRegW0m7OJCS6 z-a@OESHJ_O+dUrIpvEVayo8=HwcL2bFL4qs$&?w)*E*Gvd(6(jC{Z2`NiI?O*A zH+~zD`N+3daF$+>wSSk{semBx}7n0)G8Cf3` z?1$XFu+Aw@ul(S&2fym&2PYaneB{@l`LI4_FYaknt^!vqNyJS> zQ%)cDtDCZ+IrAqELg@mH6>WXtfVvAs#p_>?cM2!`8^BC3O?CN(dI-YF$N$i5Dak3Q zX+@Q4n;5tVW2WE+jGXvi{BKeJ67jfGiwsJGJ*xxcvsFf* z#Au{rXyS#Wl1K|0P~B`tC*a&5AqjmW9vM>7L*+ui${-RYvA!A>N$+jWy#1Uq4xrJ+ae2q6h5rZ(yk zxhskt0k!adxe1$(&Ri5%fdtOv2}KgCkuv4ZFS2XAXk$_6^FUR;6OW3x()fbfL=f%n zBY_T=*d{aveG^FCzeyh(IkB*aZIK~oVIM@}S$Tx?m&ynd)GMJk{BKErmFP?8ts;bI z#A$a|7v-MN7-rxho|Ox$aMwkAT%^3XIMO9FhWSkmJ|iEyQ$8aaVw&&aE|9txwG#Bo zOnd77n?jjdPQ^b}jBrMxa&ni61VgLHFDOYhi=0`noE|((*Env<*orCX6q*(|onvhK zF3gKGIcCU`i5<-e|5tX-CoQU%QJRN_=B43VKq>sC$tut%J5NvF+>&BKop&{ljpem1}GkF%v7MK&UOD`I~(Z?~rDWh5?+_%>suFI6UJ`jbtH4Ri`6G!+kEhC^_+VJaWe zx8Pa?WU(um=EKrZ<;MKSk?uJm39FX2fUjeSn}IJYBq{|I=7k6nT*$pi&i*N!g~bMI zW)UYQ9c~Y4r5ZG>t;bx1z*szBZ4r&Pkbh)5b^G(Ni_JZ>l%09+R=4eY+r_ADetfM4 zjbZML7Sy1VKreY$e}iNINA()C>>d!S_MBe zVnS~M=dqTKp5<159&;R|{wlup4;#wt#i}39)Tzz30UsUkFuHiO4ZK{ie*EP696KH) z5^LZFN^1q4#Q;QSZ_%%|U}LRB=)U!h&b2sPje;v0aN%s63`dfr|)yf1)hvBY-8edvTt3vFqjL#vF(5z~6^QOmI zyBgmbG={kiG3R*hqeMJ*T0qzF+8sA^AdwL8mCzVw3oubb4(?0_Gnu=TasI4i;G=AO z4+`AGB_;G$(tYH9QPGiW!Jy+KKVX1V+^@cPqz{}VpII^fs+PA!Vu4-~{%=;w4tRaw z*=I~-9CY`5H0zgvwO#dE=|~ zQ+Hb)*OyJ&^SS-M-wb?e#|rg|)g+-eeELJ~#-eQHj%p;!UGs#ACk*H^7(U30tXGVg z&=8Z!zueJ$93yHH%*eIF*ZuFUt08^*-A!#z?{@g+knQ`Vi48zPqnPC-(Nyk(BBoqo zg)C{G&AROC3ohHs%i%{*mB)a-qhI|bjzr!#`8>*TS;@Cnmd`DlAIA;+>uoaGKYEY8 z=Zd`;O7_0Uqy~*)o=pDd6#2K6Y<;PMsXv+7H%uNo?^uKhYVzz@(2>B4s2esm7x`qp zbDwW|#QVU^Sz2D&%cixpsXqQTmQC`lmH8;^@ee3vMOhXDFHt>xs>vpyG0eGDG0nf! zmnyQ+pYo-dg}u%23FuxT=2ODwOo(0bwTa<|USIn2e#>q0Jbl(5*e2bg1FQ*+@#*J8 z38vLcUko#}MdFdAs-wFG`RxKfd2iEK#hb`Pbp$dF)JfAlCNx%+?V$%9J+QgdhTHur zn$ix1>IuF^BqiY6rYz} z34OQd?uvxo@TU_06|^(>zm`qRhh4go9W&>(8ynY`8xuYMR}$a*5V3zCekN{rQF_B? zYF|H9WSYydjRoIErb{@v^oMyv9$WuMZMO4;>Kd~@bD6M&hU>@rP>yQ9c+LT~t7;?=Y< zqncV;aSLUgdx{n3!@~aUrQO{t;6V}6KJh@Yd|PhN7k8hKgyy#CqrSNRfB|S|AgN3C z_&*bpA}>*AvWjx3?pRFo<4EROU)f+(q-8%JX=^g=E~$yF#pp}gm{px$uehOO#Fx5^ zI&CIyd4er_^}?pjUu8#fNxc#pt5;CcSV6=H^(|#vP8_iMH=8l5@eg7jgDp#F3^R1F zxI^j*xNPbh^Y5KE^I%{mO_b0W=Dm==`;_e!V~Xw|tb6t{cdY$H6UXB3(OWB;J)tp7 zMoPK)f*EvR&Xqe~Gxt$k1Xy>{Ru^yU*K($G^O!Yg43h~@#W=?Dv?=~F&;RM6xg*$# zw=XJrLPN|2?2j}I9lgtw>zVPgv6Ty^#by|Wq$HP)cz1(sMg#i&>P;2VD724Tbl;2Z zEMg+(s&stLx%Z-&VtX&mAJG`*yA;73@(}YWx8UG%fIYri^YVVrtncllRnI+K6h}gD z`1L%wo$ze-TjpY1wBNhPb3$X72O-clk+-&?-@$graBJ_VnY5;l zG(MYJ8pbgVC@!?FZ$eH`X6*ho90##&^!68j>-n$SaT8en((djoHlF@uG@Xep&R|@S z+g|>%xK7&cn5Oeq_*@-;%tbmO>!vFS^vjFnrP8^BOy zjG>;y2uCu~AnJssq9}JIbvs_YwR=bC0DPzj-{G-N&l33Sm1` zB!P1``qHjZur+8Zi(4xR$V;QhCNv_G{FJ7qYVyM!wqf#HU#zSCWHgn9uldV`ms{n4 z0n2XWFvCY4)l-o#GcP+-TpXcTlky1Rq9#QFd#aCN!!cR;$8hI2QkOwpac-xvxfSVy zd3*jL0TL_#_j7O4)4m%d%le9}iwgUz3pY$~H72747e&*Gyc{U%P>5=*|X6j2&Sob(}VKJkQC*68n73cJP$ za(lX=ozS>>_$R1a5L`l1$>7jmZbLB0tY(y3l_)xyW#VKxg3$gCf^9`y>r81YHz7-R zLsJ<6>n+X)qP>Z#bqvV=tx+qOTqJDi?ep|d6n8q;~>6B z@?8urxfAdy+Anwp*70>eE^R+SfkkD)$t5ou?OkbdR-Och+Hvm?oCU}vw2SVJKgi+J zPJS$N+#2*+^YU55-(T>9O)!-c-si%ZNy5t#8e)Ek%9lUjr12w)A+}QmdR(HFo+?Oa z4Abvvd{N|6mVppu{F^mqFZ9Pg%R(P7q}vjQT_!Zd^auU8pWwIq3yRz}KKLcxIe+2MNoeTS_azRg!gw4IqZ!k|w@jE1@yW z>xp@=Unr9*JOM1DLn%KJGya-hR#^y0~YWj3+MuD_ly(}41zi8y6RPBB(gC< z_pH*%rD{+fVECVr?(ZnG!deOmn#dILBbrW zQ5Xsl1U4dQg4tgliIA)3X^}(V%vxRJ|J@lZDPCArA?DKTR#N7J{%ZF!ftxC1WPXA$ zA8w7a@k5pg8Sj)F&NEEgV<`(~p}Q7r=$BRMX9}gMWK(T16z%k>2nexvz7W6xpNKBctphneVE{%08 z2`yHqb;JfdD9j?{JwgI(Ni!X?a=zN8hgrqz)`mywDUfsN75WeRvP+od1cWipnpTgF+71Vs~g zTV}z!pX$ZW`fY0)?l#LzA8LGS<=(?@X&HU$6wDB<f_FMNW%+)hpNr6N;?-bT6sVDykUNR!x)DhArk^qMFAGk-EJb z$+?qM8o)Fq%a5CeVzmYGyfocI=O4#`f1>o3(Ws!fq$c#>{QQor#Y+PTjaB7bV0QGS zLE(tX^{WP6avsJVX?Ad|e9>nqIOb_ji|DPWUf|rlBk_8L+od8;`1IB7&lCnN@wXH%qmC4 z_TqkTQ?Xc+m#&bQE{3^Aydo$p0N+^1xeRKcTAiB;xjVs}PWOy7Ss0tN!rA^m9d*#` zy}7xOObZG%|7N6H98(N*$SN8w<-^0yC-=c@Tc4FA@GA#9a1Vv^6Fu@c<+yPskUyeL zUL^PNd_r~CPP?No`^wS`xL1X|s5+3Jj*UuA)pM^E$#bh4ZfG%8{SP^Amge-CclpRP zk9npS`C@;WH1Ta-B1@$8(1WP%*-lBM;)|Bt^Fk6@O%w+yWHgnAdr?TLw1||5ll3nP zNzfIQf{DYuS|rhrgP+kMB%$w^c4#MZhZRYzDRo=I1liI9f@J7@MHd$&Yz7K8j%J~u z{>>3$bI2K?u(ot8<{{QZQ8VliX-`PIss2(i5c+AZKezT2^&lPywPKls)%?bXeVht* z%0Ki4PyQ;jw3}kcSa#KIzB_)2{={1PRGT)BIGmq&PUjKlL2P&k~^Fn#9d7g z&9=Qg(zOan=;78Ib%w}hW~IuERCJE?u_d>r(ia@qpcRN^sXtXr6Mb9l!!ACD^f9Ki z6+Haj_np5R!ZKmck`2s+#`s)|n7QzSyJ}`yyB*chn;E*v?gf!%4SM2xp8X{gXNrCW zWhitHX70=P$3u5E9`w!>#|SluDv+3(%H&3(ZoT+VVzmqn(@j%QRW}502C};CN1sXDxac8f&rMVIO1(!6DQM;Qi{MDdQ^ z>O_uVIM?}Ou4Bug&bgCWgT^qQWlyDP0H34TPJCDsMtXa!@EJ5tloErczPK4e5_+KN zt<+ntIeiOMg3D}Y@&sviqT2h1?MyMIdUgAfxM3fHLubDuM8SK=k*g7E6wWajssEoLCU`;BKvS;_K3>Vd)8>m z57~YkP2m2{$6Pi>ED9U&)l2KN&%rtgCNivb?qJ0&;CWpKGGMqrVlms?l(C3OK>5Ww z+WdXCU)Q4VS-!p>_~qe(icwcsi0D~AYLDx|X339U`Tj^_mFW>+yIj%wS7-`dj>~uA zS1Y+Qi^YnV5aq9H!DtGvv;SdJEQ#vzoJW+%)rY`TaBgnLjakpf*J@w6q+xAA=R|me zOWCu}mL7ie^i8X*;~Rp3CMjs^kp-4#`rWDk0gR<`ZTf8284UzQK{r8qHx z6DQ^OD!J38{$cCadV`+XK@iL2|NTr#c@T?Fr6fT_t?M9*92P~#@|u_{Sw-K(v8$|h zAvbqF`h6KMl3Pf>n}{UTMQ24efyBk=C60A*f&=N`qQ0<&NcxKEpSy9te9u1@vtSC1nE z2m&vULweqF+mXNz>#3$0`cft1h_0W6g_I=Yh_LXs+89Z{>>F`yR_h%$P@N>-c^&_d zPoXd+uk%NAzV~(m@)WPr{Yx5@r2eCn-m1uKv+f+Ri=j`LT~x;CSqJ>{NFUDzgv{(x zh@L8r5YK8i6Bo-@aci*9QKxHf;R{nSeQ9@-q+-ElByH#3{RQ4;hYsQ|#5`B(9&Cm{ zVm7OLp{%eHN^kc+AY`II*>*dDu9lnn$YCc*D&4lwipYlW&>8UOekH_Ov#Y%JE32)lqs= zK7aMZV~_!qI|RiTKPp+`dw$-erVR9H6B^pE-!o=PTO$_V%VQZjTLZufB!K%^T0GF} zx|ZF$jlfbGr#Q05%&ZcpqolQoHCv*KDuLXWi2C`z$k|ls=Rp$$37&eqX6qM&cIpfI z7trIuWes|(IY(;up42=jrfN^7<7pL>o6s2MDuCws3SEf&wv6R)M{=s3fkl$k0@n9ug$Z@=EiB05f&YXtJnc|+k8K^*jCy^ww~YFW`wOnVqD9U^@ZjXl!(Nt zGn>`92>lL13-r218mi+?Dym3AW0(&R^9Ix#n9b~LeAo_HHmv!YF89^ps9<_nz^2;g zJ$dZ*4<5aC32o)2ct7r7#X;RMEz|0Khs@`-v00HiV*E1oN~S>Ona67iUU%N-s{4`B zszD5zg6Cpn)w^LcwQK4Kwa|B)~l;y(QGfgs3lOZCD%kwXjfw-ta>*=vc9Y zWw_fz_ugp9V_zSRnkUMd&=}@sSm<)xWQH{TS`8nzK$Ly>x4(89+=XXlXF%a9tfETD z%GK`496B%pY^B^(hbFLQ`ptG-u0Hg!4NM#REQOEyN@!TkM9jFoO<&sGH=<`XXvoek z;K#Ltb6ry?`n!viqK(Jj%c^+jg`yM47baE0Q>=cUOJi0kR?D}>e4Z`(EFKV9mBt(}|FBhi%^8Fa zo-gXwpfSvWRNAB2vaunu0bWBkZP6(-Z9-$1`?D=Nq-leaO4XxoWqj74F-#xUcBx&Xphx}redqkADL+!u@+nQnTqnIku9j*eB|6A!EZ2mAF`FY`JX$XF-&Hx zQ~xUPV55~RMX!shm98t#M0{%x#GJx*US9g-p%!`z&hbAXPX zH=0e#M}8#_uGe|z+-XN{y5BhP`9aa{s$yoJZt+|9#u^8g+&dzkVoRggx;z0)dZnT? z99D!^S^M{Qu5#J7er?a!mQHzETkG!{rC^RkkYGY%wY`X#$4hMkn>y>rPRV+$zwl5b zEK&*9pfOB(C+=Hml|Zk$ojv*Ds?$a-y&*7@=>3GoFrOjj+@i-X{T&O;(|SyMeoiBu zsJx{^xe1M7a<{|1EqsQ~B^rrI;SsOAcFwWmk*8O00j@z~n0^l@?>WKNS!a}H$D*Rk zdrl1+!yJWaz4n~o4%w7aZX(_oeCF44)}S#=E)}>>rS5^=_zZ27%9Nb z4(z8hM6Uv7LSvXK67xi9_n=F5CQ1m_KztLla5Wv1S*YmUT<}eOl|7Ek+@HgdN zP`~fJBG$DrtK!ljm-ulbM5-?4gUXE1f7g%gcDbl^a4Gc2Y&L11{;%~oAnMXJXpHx@ zxpMWNqKws5%_I#A&jJ63^^1p!gvKy8118ot{K_Z~bRgX~GXRF>5m7!Ht6DM=B{T+q z23fvFRIO7?SoK-b%gUj7s)|_(dWhGMc`08-cyHiia#ak|5Mw9M$E|OUUY`$_9zS7o z=I2U`cS2*Bf1?BkuAL;#!a1jq}X7rOqukBOAvmUeRppJq%+o_H+*(3w-A<)A*uoV zmJeIMjN*Zkz%!{0v{IcE*NIX__AIyLWvaaE!cyZdVxx(^sE!>-3(#eJm3|*#rzTy5 z(*t?KXMO(sE@M{oGZL?~yRg>Z7e_p_s{gY^U`o}~??(m(`j{PD2g~z@I3PB;sbOs8 z#o7sgiJ3KU-9^k*G*Z>5-7O$faZggEshDFMQ?5r5Q)Oo}HS5^3$V1dm{@B@8{k9M6`f8G-VTRE|QOKfRTZV`=P{zx7! zVI5;k*|uLWGQ*_;=RfoT(`C1!tt#Z4&=_WjyNWSIZdlgo-SOW19aiTCOVULW8e*P` zlHJ}M1Lu&3xZ6ovBY&<;jS%SX-Vl8V?v72?h2f=3dh|CKCG&DM|DzTA0E+GC$5Y>~ z83zU}UNMBR3JVaezEFM0a`{4-6UTEz5L*p;D{BX|0*-L<3i^t&0-vVZ`eJ2W!m<{T zvTD#;SDkky{7y1rtt!ummBC2cWC^prYZ z`+zA!46c#2=(B#pSs;;O-`3NmlD^#cwbLL=6?z!rbW<`HD0dqZp(iw2mGYVxB|GZ; z4eRK^K|V?4+g*>M3=B;Bx}|2p`!h{kB=6{LZ4Kyb)K0g64THYS0*F8T*>b=tk z4c-Eni;9&E{ZTApbm%tu!KfyI!c{PN)2;lIWQ-lMVNLMUUdH zYa@XQz(%9Fr1$L;?>Z;+h9Br-e2vw~S>6n?&vhhJ%XCEg{E}6qX7ksfBR-h&$VzLL zRBB68YAcRKT3L9fG89Z|Xl_7uKuMnJ)5=xG#%oRUn`Fd6J{5v2f5|HUDC%2>Equ0dm%C17^+_M54?i8>6DlEZC<{>*jm zBnr#KsxPiiY81?;{u0_RcL0KZrX+c=Q9x1s2#S(xK|s-#4Gvu%+f#$YBoQ2ebWT&T zaI{^ErHY)G1#!;lgIomSa+* zAm-V=kFq)zi9W0t+LJn=sl0LfiY)pikvQs9RL35{X? zgI;QHY5vfmRa5g!JiLcCFA* zUpleJHncnxFhg{CRmWG4Rapz}%%(`D8pgBIo7e(hf{MX*SFw}|wp zPSq2Ul%32NqD2rls594w|I8;aFBLPlgvJCim`5@fla5x!jLV0- zMkOEMYtR_xp~UQ8?7TKB7D;Xi%&x_{Cp3onEHTd(k2OeK-Q(;pn;$oOuNyW6X5wom zG={kd7YFiLD_$4!r)_dej!f0F+eU7_(>dJX?PPY1$(vXMaOdT?!9f> z1ZClBct62`YKOV4nHGIPZA3z0z%ez{7uQ=zf*UWtz_-vB*GEW#gP9CJCX=vW5)X4W zRtO~Gx?tG-Zp}fQS|tlC3BBQmHP|>)kr^w&mvl8AzlHeul6qB@0rv!=zD)R%}=}jn&PnB64Cf_?fJ3y2yp5RvN>6gH^e(*wsvx zBv^)t+w--NVpU)PmBug!4FKj@q8deUf?lf5iE}68e?()Lbtu>^Q5^12hW|4wT;=3S zLPJcNJ;!aXbZC-$ld*Pi?}gvL_nR&EM&0wBMF%K9Ey=}*jP8<-Zo5*A$*Lq7Tn8l5 zDnTN;sAT7b(;a{FDPH^Fu`f>?GRzNf-exaOn|eu6uT{3WK(I@K${q*4%ZxO1Khb%N zw|f$L6;63s;{QvW&A_q=Q(vlt3F%L^v|*S$+@~Qu!52_}&~iky>p)+F#xS=9Cc=Ig z7SlRq4E8u+i^F#|SJIbum!Ev7L1WkyLLm;t$jQ;Cr8^U!ylHL7&O4W!jPTV~(6GJKI>|zzxKB zWMBM!g+7~qb4V#W`=J$%`KkS!(yY9k5&W(II$3h3n zwE7(bcXd1~3=s)TCVlzh+fyjeS>2g*CKINVx?(*>L=!)-!in0pd)De;+t^}Y4D zwp+^Atys!V{?nW*{`J&>kV|r~BB3$NJ-8ZtiSRjreTxrsV6x|%mOtg;A9zsZX&H}d z&=}^H#C%pddZ5>+r0j<6N88o8nQKd{@+TF@Ut<|55 zhP4jGul_5Fhe@g|x9;^*SHJh-!*OkL{z50u+a&fmw`!3GB{O>Ra2baeA6%5gKFg?v z6q4crGfw#;={aF|NoW*v-gcDj9JW;KItV>7M1-8%$&i)MD5g&?tM|?a9@Gno?hH;a z6PgP7yMK!&2TX~Z!`;c2_e*8vv)sNydml%+B=pAf!<55kMSNR)(*QlToo|cP(QFz% z5co;xiT@n>iu(f_@CtDaf|Fy`KwsKB%fK3~k1A$)HsCt?W8}m&uM!;AX=?h4b%QI( zM%4+Yx`S54Kgjm;(%A#skd^<-=PgKQm$IGE80I*7A1?@IFm&caDi4NPgT^qoC+1Z} z@tQj60*)EaP?v;;m_E1dU8sV)PYiOPSN8I{b1mWy>25V>4AW;qx?M0sy=-2mixS3X z4I0B-4O;H*`~^O-ey9USr+mt0(>sK_f_10sSz$tT>nt)0+p!&EwVm{^)y)uKR^te&^-<3Ykshyw2`> z9@=H+O-k90rGI_$^7=S7IZG6={0&6bF<@*<%VFili4A!qRi0fel&8mjvhNlhu;`xi zw<)<+uveiSGDjRxvLTJ~yOFJOHAt1EN68)^+0x(L_s*>CFx~jw#^buq8*>8WH>W5Z zb2?S&!SxVolh0!3YZ>ePgag?2@DB&QwfnbC%WYnAgM`ar)O|i|b2`&g?T#5ED=LC{ zO6z}DMn$$5`qO8p_MTF59~SE>!#&QNuW;gwqk0W$@*T|g>)WHc7P|#zy*mA^(y#R= zqoMpfCU0@URd3DXThL|w!VVvn2GL&H-BHjD)Fl*Lmmg-gjg7R+CM4Ew+psQgLuzZ( zmG?l?_(%5mW!Wb?jo7u6?f3LQ2T$5(PqbUvi?Dnn)Zahy)bKmt+Q(8Q6tQhOD^SzqW1$<9&*4FTz-+l=YcA7kOXzuhRB< z6JPe%E$hL+_IG4TEI!mpNf4)hffQ$b+d$ICeY+k0!=Pvsr=oaJxO^}mc&Q) z&EXqtx9>4999pD%Kv2TK2@d_AIY8PD<%3z$VV?O&Cq>Z*ma+wh4qoP-*VX~^e-?oR zy-jF{>FwK>DBevF%wR>u*X&-(bpZ>T>F|}%80PoHd<>OjW34}OmbzonTVK*aSQie>l(@A}9J**20X5;7og5@* z-aO+RIHJI~e$w1{ME!)uhH;IK*=vF(Nx_H%*Ow}-u0I(K@%Lm0`XdC{(T%xr09xGN z#Ok9i4r76ZDja9s5&t}*G0+m#bB?Hd*f-a8-YE51Gn+A<9|@zFg01k?D{HQG*9tV; zFaeG?28*BEgGt2Zp5y~oU!Z717CoaB-Ki2tXdmV zOY&+39wbs@@lrPZu?NPz`|!TrBYX%p$9+!O*40mFYnzNp*7(9o*o#fVtqnBBv8*v4 z*#%#BufJ$#IPNE2veG^SpILfK`d1GE-2EG*9es~zC?g?U^EjkcZgULu4|5q^AIsjK z{`$@*-L=q``0`SGH1BQzV|!>^3qC@o(N^Yr{R2zqBfEI9)*W7&I=qzifAg<5J?Hvw zNe;gzL)%iWxeUT+X{#R6AUuS0jac0pG{o$|BLBw6WmJWD>%!JGcldJs)8C&t6_{s< zx{M|xe1v-?mBuh{pm=U9VwyjE?j4`6K6DvkURuOVXsmT;!8*FP;u!rDcEO-lQtR0o z^uk$sqE-nFVNU>Jc?njZ_p1OiEc3}A^`cI8xGmgIL|TK!Fb5O!u_C7QE*Puz4nFmz z6&iRzGQmt}hW^EF&z{?C5N36=H@@tJL5J1P` z;TEs9@}$O=DQLU$!VtHAy@&u-FLWR-{XCqn;^?1X?xHA7Qmi&6CNFX99p!Y)z96S<2taEe5l2hm!a@qMwt~=P|qfE zab6-#jG53FCS4169s5k|Ez6h%UqF~)#@Nm+LH&>w?p@@9RZFSlZ2Lwc`d0E?&$r>$ zEcSBIqfqtNrnHP6bNr5_?7Wp8e|h)@a1c_=gvP4!AThsT>%}dsfq&U1&)u~36D49M zCL*CB=667H+w(C;F#QTAANf7JOO~i9wyFFh`CleaO zWSVI=P5MrdgE%KR4h#GDQ)H;)lTr#FF?T*VH70stV z8BHjH%~38kpSg=V52%8f&=B(#vi&L_uaSqi@0ky#`|wrzb3#MR#n@VYk9Tpr0W%Dk zx=5Ve_H&n)|9tRXzVCT@ZgzGo>L(Gn2LSVdhH;QcjW4YvHq7+in>S&!9NRK!GIwLc zXX5Z;dZ;f|1~>i5Xbjr#!0trdk1@@Qy6(6YUYh+DD&cqmMFqPF4Ke9SxC5zovxj7WngZZtvX=|0-@nXV?BoMPKGxYRhg=haY>gycoW`SO&YKREa7ppFF z<*I*BgWjqb=;J1`si08meK^eDZ0|`sKbwUQax2W!^*7 z%FbI}=?#B(Knr~=b{olpE4fQin-Uts{3m%}vbu`iCepwODxHesa1S*)ZoY(um_Jf< zU8vEwNDnBQU|?!~hIaQy_Q-ftHcX{4%%zF>6VSnD!0fZX8+687+mx~^e|U4Vi92(I z%RRXoG=@2bn74Oo)iQJ%+UDAE@7Yi+Ml{f2@|2!@oiMJ8+oX`{hcO;zzIbgMlK3Iix zt#jy0yW38*AaJ2S+2*aG4hE@PTD0JJFz@)Vo6@pzr8BPE{(+Sti4O(yT(ZJP4u!j7 zr3Wlua`S{F*aEQ~2%gcFuwu_944q0;B|zHz?D!k5-?#qYA=QdaXbkgyVm=_4fd@Nk z6>^s(d?qx+WFz_echp7)aoQ53iAU+5=SOqML)@qdz2OJ?xWj0F&W=8Bv!<1Y^VDAA z#wIj|xd#nU;>Jqg2R4oGyWpO2%dP@NOgl_M1I+2;{`ar_o8p>qJJ;}a@}*5)_2$=VZAwp)pLq%l(mbsi51^1-C0@N8NtS#@j7-uwl+?rq1kzM$4XmbIOSKuE2iV zkh!f#uh4hT64>-J4Nl12He!o{Y9HQm%-^;-0k-AFSL>(lw!EYy_tn~zkelHnbgw`! zi%KhF_B-yU=Qb*D3KnwvR)dDs@*DXK*qI*PJycgd2m^zESYl9n!LFV+2IW8CEC0y! z79M+XB{JB}twLt`$SibHUsyP|S$sT%TayB)i?(Id{m#UBMzBY^b(7E-CW9Z`+C|K@ z8YZ22`Fv~?Uw_N}OMLlHO!wTdqTsi>YvJBseYsL8zb$6F(yXqo7tqT7)WaMwjA{*I zwfL}`5whW@_WR`9b=IvCKVgS=q~*8vUZSu0A`p`M3y0u%A;Tt_sxnCVC(;Z{sR@lW zrFVYw?X>;LCLh`I>rHRIYE7(6Nq6Dmh%{mpb|4c=#JxEkYER^ra1&tCBHAf)5X>B}3VxjGir zNQViSyD)iCs88~xm2@ym@=6w$B*zf2Al>`E^LIOY(k5uV-=QKEmk{$?>id6^L|ClgvK!a&9r}$ zF(I_^oGW*}X6~aKm$D!49ohQw^}Np^!Axij^FGSq@*5jZ+j6oYolvzS!nEv?Z{Y72UygPXtA)N*+4F@H+CFV|Y#KSd?D?9AB)VDP_ge+9F zU-G?`_dew^KxF*}0r|)+F%H@MY00B5Ie*F1e0lgGx17t;5_a>XozNTpi^RXF)3%g# ziX|bVRRuHXrM0Ef7^a`(uSWWK(@r)+V0W%OtbWUN-VmOBI0C~g+;Hj7VG%*}FA6wU z<2@4Gc64h)rdibS309hqe3cB#LAsid#+a!`pH2S9t*amR)M+L6IBOo~LY3YbtpHiA zSgbohH1uw^1Rt5MDCA*9NNMEyp~-<+gT^q&5pyxp*G2RaKL!tHz4q6pb}UV6w0q~d|7D;4VPfAc=>;u~wk+RH7mtIYjX{n>*+0fI9ll?J(&EFiTjjSU zY*j@Rdi#1NdDHjsZc#rXcn^0d-oW-|{0BZPt}}b%iyz9@Y>EVsXtAJ_6GbdRfozb~ zW6BlehY5L@7l<2zNKW1KLyZIa--_fmG)iF8%7J@aJ&5dY!hL|(3mo$tS}Q>4zZI3N zLBobz2u%1?Asvg3lYP~CaVTPESr5fz8V-yL#iU7tJZy$AZ>z1=IPmV(IRP00HY5Ka zw)y5^>OPp;TlfvWuQ+T-^C^J<{k;r4zdrnci%=CN+@`f^`>VuhCNw64UBPRGt2gg> zO8O9RWf>^!g}Pyqa|a9OB<+NrrJPSVI+S@dTA7(V{3Gk})J^Sc_D4*P%sA*i=b^DR zXbkfL0{s@4P#zlznvkf|;8fjMydD?%S*ujbR=|K94R+Sc2s? z!$h>#pk)tPj9vR?F+o8O5*lJ|0q)B4h&hP#@qi|aGCeiKc_1)bSeLlF+g^gG#G6Qf z#4#+kDEf=7eb4IZmZD$|phf95@rbti%r!LA=oXDM&ME66mpcR0AUTe#@?oW$ zmdbI-sH0DwRMgw5DlA7QhTJ&<9gG+N{9kVJB@?BWly`fU5J~L`8h`x47RxfVuvNlw zWJLqpu(}@Ob`|CHU4{>fX3UFS$agDW?2@e4NYFr$xC!nb*;iluapggeZClFz{_lOC z+2OOp&_EIeREx$MeF!mcrFp9>_a!DD+066aU-g0q5Qp6V>Z7(=q}POiS%bzfxjNxy zPy_0`WJEmAZ9Zf4F+c8J%9gsR?~J?tatbi>x(5*^G{p29mpc-3Cp;>YQ_S#crEJxM zhkrNzQr4YuFlgzZ@Cl7!b_J>HM*3LcZQwZIlxzOdcElt<+aM3+-XP_XisoiGT4kK4 z^@AT)yei5|zPIwurmEa6atZw1-PHE3MSFs6%lb zY!-0oxm$g@^3FWGTrWddC^w-o%)yk)^+mbJ76vBsvf9eqH4|AUG{mItx&?g9!2^5- zT^Ak@3nhzy@Nbl!5pTF1ozb`B8ppOE^cF&{@u4#B_9OHA3-e)YwSq1=WXxJGtV7Fq zmzby;G{(F?&~}d~U*JJkBcjyVKMH0I8pB+J0{Ou!L)1~b<_YsMIn;nd7`Z^o)YFb0 zxiHq~7IjuR6}Dz6`~IMX=kGRa&yq{nkm?uwPTwv*vanI~C$Oz2L9?0ff4u$=*E3X) z)18VYNNB8TK6`nCh$}1x=_0*dd)4lKW;1njex;U=+$!R_3-Q9bmE7CKV%7gPN8E~C zKT@%v=5n7L7GB$)xo6uakJ}Yh$uVot80KjD4BHmFmw9u;Umt|S4m$yJcCmXUG=|Bn z{)MgkqURmWOK^Hhq*Q~(p#KI^_Xz1@YoJ>hxub@FMwNUJK|*7geuHusY0W?{KHT6^ z_U_VG%>C&|E;uK#r+N) z@BWeP_R&|Xe||SSF_{+bXT?VSL7lYhd_RBP&x#F0yC~6Yoy|o+HIDF?xvrf2=irH0^DD;_x#xM^g<{fyT zrGq#%5Fwff7rcF0k{Okx-l8pG@$pZ7=`1&y@e6FVlKVkR`iWW+{!39y)M!|UEu zCd(1nwN!dPM*1l4cuPK^C;q#H?un-D7~mR_PIzi#eQPVOu)%2(I1y;SFa%k5(3dJZ zdHRzb{j)kS{MKzH?H9Jb?YvmaHs9QE*lkS4nnb!LG{(1|5WOeuHe~8Z>L&JJ!`BKlJ62ZU2NuF=ZBem-MlKDY2ldl(HVP|98*lo;pdAL2`n^tZ^Wsff zCClf}1N@YR_~qr;l@AwH2wmA$7IY<_-Okgm#wO`mZng6^H9n$*>?3zraS%8EMIsoO zP%yDU2@Ns*Lb})Ue}EhG-(ar~8+yi^VW$r(Wy2>vu;a+78v%16RxXbJPiTmFKD$IC zA9L^kW)QZp*C(EUC%fPYjq$k_ig#;^fCHbluBa1>Dtsn1hWRx4T)2qYGG_Z-S8K*m zsfT~w_tvqiFgh&Bn;?q5dMSJRo-x}_Yu~cu#DuOhn7P7TPoT?=I&CK2OI}yJ zcs%=ZWH!=ook-^{BLn%9BeMyOiQG?1-WHJuv+={E-OhRS)4saiGXR*$Y$2gB%yTK8 zycvUfTK0km$4>hsyM@=8Cw^!`W0>Bh&W{Uoy}ryxc6HBLufMh*j%@$!)6GtP{1~nw z<#pek{fdwL!1HfPnXZHC($&pqz%n0M{miKohi$YymZBEF=EEiK+0%>M9}n}(uRTS} zN3&XdD;g}9}(q&Q%qTr&WppKtzhy%kn0Y5D`NIDBM}p1s}?Yp(>KYm-rj zKYr;C`}$U9RmzvJK4XhjkqHc1d9B-8FRAO;1;r*iSqY6+rTj23I~KuRC9b};sXP`b z>|!y=fb*GNDe)OvdsWoE!Vn}hhW#b^?N;PhE>It$yX0$t-$v1(Af|-IF#7<`jVFD~ zr=8W3foLmWCN?dhG0b{kb}Yg)pm_o|UK?jf5#xz^UVY>1jAn=uKEVQvLm;SMTlwt2lW%x?qCYlP3BDG7~X?n}&DXoTu~ z)(=Vjaq?pWwz}*T4BaCRxMumQpW_{EdFGdp9web5<`CLF?^Rz+D-dg+rr@-wr1rGQ z5*lKzfZCMTrlnX>S^iZ3Nn5Ih_E18O1Y(ejm#o)2n zlK@mcm4oETQk5mx27H*4p83$jLssOGHZdYKXo%^XaWHX~lB!(GcRnoDMzk=I7*Y)? z%g?G|xhRiB365D|79>}u`{CI8xl0#|wPjzMzbj^F6^roDV>)hKF>j$*nUCP`QoKs# z127(;1ec|B;&fk@?PAtCJ8ib%g}g8?&qfp!ETN$UJ*GElw?Zv|Ne93`GCT11-p&K= zxZ=#?fSG$10duKOPM&ne`q=+_;^C%+`#gDUNv&#d2k*ajy#u$~2McJQJoxl3mma>O zA5na<+}&`CNU=dWqIy4!%yenrimgFoRbhOJ+rMc2%~r{7FOK49)t^W49gFKKiN>^4 zM$aW*TWS2-xngOuQi$W(zuB(K)rUTY)RB2<lF%!D6apGsd|B6q-ywK(zhaOhTW_I1;#XB01a%%(8w~(bm;{33i96v;;6{6` zwESsz{=eR?Jz z^?KB+*CioJTf`$(u8Mjzb*XwkT6J(qt8sJDh9Vlu{eJiQt+n_0?K4OE`RJWLetvS+ zZ|$|$Yp=cb-fQy^Y7&r}4s)B=h@+mxmQiWeAAsPaXnUQGcy#i;KRJ5~U(SO17I8oJ z_iAB98LF|XGTmsCz*tw9rgj5JLyYs!kM;i5-RB-wC@$J#!1q=)vnvUE5*Wkz1L9-` zx(cTb9mRd;-*fVn+bpZa@&v|k{){;9$m7(lPT8L4t!wo?fiawq5ohL#LiRvBCib=H zI=*TDLx8U|%k|74@ah<{9M}j@okb^gV2G0mqt$zJh*&`ziE|#d zw5Bg*H(QJ(;MAY8q?zmrCeA1c72{O*o(-#vQ~Gl4OjlZbPA9;a4&D&BD6*A9R3T`Yg5 zbS5x{)8}h9X>Mt$S*$5p1FeBOfic|QCcTMgk&p#Wa@+Ak@9KXHJe%rnFvK~O zYQKQN!*RlcFg`TsL3gc%Qg)XL&N?uL(}!RGDREH9oJf?2J6^PdOxXHmDHiLg^zJGZ znfIvkfi>X6^kaqS`D6+o*<$RVaAJ&Z=3lAbcc7=0zO4&7u|1WCo4Jq6KQe!T6F@qH zc7vp!fegz5sLNMY1Q6R;q>->uI*ct1$D_);guS@l`H7V$ywyB``QGUCMX_~sqHYL5~)97B(e&^2SVwHh(p%~|fHLNvB*MVX8 zJY)N@dn=@SK{JdU`eK6cyFopC9E~VgM)S*ZZ6YZ3f^!SR;{~DVUcan34vr5hJ%{3> zN&^b8ma4^^d|^G{7p^QVoXOts5h{5u-t@vtj(Z)IlE8i6V_*Dcv+1okD&HE_&Z_gN zHFX}F%5Dq|@HR8jSt%-QlAKr<5*Wita@-kt@|M?% z!pAh*S)t)8X5--1v*1HQa{^;Hxhv336W)eC+6?jZNh^QW-3gpf4bog{#G}&Jwa&!c zfz?Foe4~6RHQ*cfb8(gJWIi9Ztt9dfNu4q?bafQ&exivB&522)OouKxvsueJySdV} zkfpYjCiPD!k0v6h17kRSDZ#-_;*+9Oc3kdWbSK*LU@bUyo{y~a5sbAkjWAv*Vw}Q= zc()1!!d{kqdAv=Vcv*GRJXE^tm=E@ZhLR7hp=a1ow#su2-}wj|gLH+x>{bcO>%bV! z#{r6x%dCE^#WK2ynh#rZwE~Sa1TQHbpstKW5sKC}tgUw7e0EFg0^AwW*(sjC4i0PD zAHx2&NISsWxTe$VvfBb04;IydF`Ryx;A+qgg9|J*#N>1`bp< zlY4^kPpHa<743LrPzudbt)n~A2L6SL79X9mtCRXyUR!h6c<%q4Oa{k_lQ1tm!XY5> zk%>$MAwscJLvprkVgU|<##v%s84L6?P$jGhT}RVAy&*?9q&tX2^~IepRv*kSq0V~D zDyHrNG=UypfKEweRM~lo>&e<@FF{~D;y{k+UcELYbBFk9y-o8ShH{iFo7vqy0~Glc zcjB<>>-KqQj!mqf1CDjaysN&HwF^PN^e6LjhLa?@i^YS4F5E6F`KaP~P+`h#?1H;U z*b*jVRxoA7^&foj?Gpzdm^LfZ7wo9sW66J?e`QBMpMUk7m`l!OWtzjzZ@X$iDxmX> zanh_NCW_;BzWlCni}os{&FX5DRheOLzsB+1yYei55Zt;Y^<}omsx+B$)qx?-dnpjV zf&4zaK`NmumMYbC%(8n@d~h8Y;v7!S+)1fqlFs`=l)VwB_)a5-e8T(1fyyAC8er9F z#NxhIaXe1zH}vY+~{N zLy2{>|7qTNMq5XB%LJqm+d8pJmri*KWLz1A;}_V`GPl&;iZ@^7mUrsS#&IC9E(9G3 zyXk&>U2?AHRfeQ!z|}gRQ(8a3mu?A^vNF60)fG|rPv&_LT`Tu}f`GtL$jSdzMW#DY zO)nybQ5aJ?yW8+3Mnop&xSz7Cqub4>KE|7%QH3o{riqNe1I-3alzt=TR-u>OCE~N{ zz!*+0LAgr_ALA4Yz}l-86RB}rIe{^pvx#%_Je?NDhI*ITQSCry0z;hMWbOr9+;pKc zu-xF!o7yn(sXL#B2TK$^fl-`(e01LzoWZ}_q4rP4jz4nqlLq_h^L{*Kim0-QKOL2s zZfcfD_r80?gHvCB}u5!PsZ|@SqJwm*0P>;4Xx&RJjowOvb&t z`+Lj3v&Q2E%F?OCJJO%lYRAd>rcP{(p-Odjw0Bjjl&Ax{qnzmMPWw;dO*voDw=sPsWmVw^??nZaq=3hRNhhmK!+&!2p-ZftXl z^P)2+`}mD58{F`R#~`}I0VOadIv!NX9p?#Jp?=XEB(|@x`B+*BjG?Bu+%-aO5R^p{ zr!3v7&vFQ=oW^PijNyEOIOzpoWg)=QJVf!GzP+Bj_q%I=bG-QAIxvQlI}j>^XLn2& zmt?*qc1dc!#ph@3^VFodJTrAdo)Za-wQvUrc2o0$lYt#|hBw%9(AKy9VgtlT$q<#m z7|wU;3Wv!(3|-xGeK~{fPOnWd;i9o?8jHXD-jrLub2l4F2v1}jteE1ZSZ#^hp z1YodMD(y0UE+n^g+PLZyl&cp_zgMSpWu2-I7-4W{% z&!M>BgwBQYW_7ghzRO+{=0eORL|`*oySgw{o{+sVVDnqT2hQkrj@OK3KY-2?NHZUH zCy>3A-Sg~A?|Lxo`)!b{zhTC`A!-wFnjh7d8aVZ*xOC=eZaw;rUBq1zI&s@fbJxz8 zUZDKSglcwlA>F=nXY-6X7=F7ZWSItd9eoIzl;tRt|G90as zN`Nchr$+dq+8$**!ifT#mip-3eGa_y=YMdX2Zc{dh(>Fu5z+$te|hFFZh03zGF45X zHG+Y7)}8VH#>%Y7WmnZq;RoOYP8Bf7k~tvqJT2_nA!Iin77W-6{p0;~b!{*{9uqvJ z8&P@H_bERkt95&evk6kH$@1H&xIDF)ovaUcTGg@ka(XkIAdlLoRe0n+64=yFfHUTx zF-r_zf+jJXmXT^I`;aARn3F)z#sAGKYt1BcIF>qjWh zOK+7Ah=m4};J^P1UNzI*(lRmzQzLOFz8Wu;vUVJspdkxPgzgUZBzB59csN>Zc05Hs z*qW^wH78$NYg}h@Uy#-UpUp1zeERk4&L^MInhD7Lr|`KBJn?%2J)HQ*=9wr8fUuzd zy&rGe2&UuUQC@Zl3~`Pkx@HcKx8bddLy1yRH#|H`D&fK`t|fNy$x;wOpviaEj$aYj zRcH^hYf%^!oRqZauAqIn&|U|IICBmzmagfkcG{#{V22Zjn7|wU4H?h%n+cEDH4FP? zVy_{%BQI)94}wgX0UnZWU)%>?NsCgdajqXD_X7`gpa>zA8MD5qz)Cwtovs6~oh}6? z#Y~k{AnttEwecDQKmubpj{psozNL2AkP_ofcvN=x)>ht}hx3g52JsFEgDT6ve2SE~ zdRSj&m4G^ew2qxj0ydkza2~QY64`k@sbdiTQ2$N=dUr7?j(KJdMP4?B(cG2iSps7u z)3dst6F$Z%a(GJPTV67r;7ni)=W61-FOSm*46N?}1L?XU_h}&()7}Fz7ECCO;O$@r8-D za{jY}zV-pxn)&u%SqTiuy%AXD1d{jF@l4v_^#yKML=~oX`jaI-l?9P2jAxf3p|y=s zEu3WkM*?Fw_owPVj0Z#&+^a6KNBR2usdn)!IDs*oKB3Fox_~qA$`Y%OY&mV!e&naB z^{;^1@$OdOuLE!Rz1!TZX;yO=Zd1)NgjG$6#H&WPY}OdiCoqPd7YDmbg#5e~RpDGB zB-ep4oJ?xEYY88FElUKB8gbpNcOrpVV--GuAx@wFe}*_S6$_Rw@VJ>n_lk2Q_3%eh zFFkzh84`Hnzlx~2+cw27d^1mrOakyTQ{lp|hn+1b%GJR* z0+UYF)H!4~5ARw~l6e(xrFV8yYq$s1(!NKGUbxlmlc)H&-t**(Bs}4vtvG7wsIq%V zWF7dNz*xWhIjbwAUjbensg+tOyI;ahR(a_J#&8DN#cM;hDMoL~DZ7XBbS5w+Uoy-6 zTIdYC60_d0P`q{fqrdUx=cj=G`v}fDFvOYL?-}n8)tpx5(m3LehOE)|yxYSr7fsx{ z8}c9~c6pH#d%oGvQGLb|KK8bv+F0(9EXk&}#nbsc)Xa!ik&wV!w*q{OU&ui8KY7nL z*FLx}YZoLpAsfxO`}XUT{%~@^^^s=74BjO4o3-a3JgfH{ubu4HYL?5Y49sTmT}@px zIy(K~_prk>87vU@h*s5sv2NXnM%+!LC+?Qf(~zAWsL?I6Y9&CM4c#(jRyJv~QmEy! ze6zM@bmxa(`_^Cfx*y#9rYK%ek_5(ZHnJ;ciZTayktoW_o_PLQ-J=UH){>C)0E##C z+3y_x_rc2xx^FvlMS6=A`xPUnja|I_IWHmaBbU%eUIr%u4hH*>;0~n-rHu35Xf|6D ztOLWYJiuP~b91bDP$d=fVK2XN{e3&VH4iu+$+IVcAI*K?%hRbp8I0llD{xl&%xPWNgq{Df zQpzV$5Qjx2z*do(z!+MdOmXw`+M$YBJml$KW6v4R?k6}C7{kf)8}76`PAfUH{mO$r z_sa_?rmgZUO<)YCKON=$Qrve-@}4_((a~d;66gJSoCyqZ?o0^{1*lec2-`1(SMf0J zb`pNpfiawZS@~JQ$8J91KkiwnjWd#?;q<1F;1uh`=01I?fm44n7{lo|hJAK8e@K4* zp1PE&eW>Yf%Shs)_J@qF5E<2hA)Umz`X_jBk8cUf3r*nZA)G+*VcUD`g;DcFkl^p( zix#?2#*ORIthGFoMi$-oJNp)PCSqg*7?Zl?Dt`U#Lswl~ zm|ReBpJ~a!gG|kF=*|c?uC_?A_gzk1ZC-HO=0OaLh-a7|QE3XL&r|g?^UAxTHM*U9 zWH7HfFe>TNB;q_#ls>>~3Z|%KTFUNOGBe)BpTHRN12bdo6ds~N*6$vk^vcV>IRC{fd?xM;7{%ZP7QHM@E((aM~6FJtb~@r~^ZsbeAs9 z3kP`dK2TMCG55-xuX>X@@Wk(pD_BcS7E9il2ih}z2sn+!3S)eD4?7Ul76QE4j=q%L zHa*&x!Whnvh;w_&qQRF_k(w&f{b=8G*>y;T(xk1v=K!awv~CRUwu+1dhn+ zOAW;Olff9`(Lk&WD1aWvhb6kQ`zKjZ`UEkuIxvQFYvRlfh2@P5=S6g1nhjNTU<_v` z888IPv^FdpwD72AzmYO8H$=uVO1up_PMp1nD}u2m?qm}qs+;&q2Uh8&?rZ3|xB~P< zU+Dk6#fF~8E>1PyOhCceDffbSntU&!z|kmtityJ_YZ_b`3g#b0t3s|dwaV^CdCju@ z>AFeL-6Qf0X(6E!WlZ$QKAZK1?awRr_t(gD&Y96QvC74X|Hit{1t$;q%Bb0_l$h{5 zfgx(zbcs`S88Bm;7QcleFk0ujhD|3hMlN?XVB2kT7ixaZYr^s4xy`8Qwg7i;S376X zgeXv^7_GzZ<=S$I;H||a=!)Bo9wP3A;K$iwaX%=@m*Fol^;sD>9Y;u7aG;^JeTG}u zS$&sA@-5r)F&H;2sKK(ns^cSDbVI2$DSd|`JvxGgu*J${hym_caQ}f%;edi>TPP

#05~@<;nf=y*G5jHHZ;+I;k76IVbi^)8m*Jj5IsJnQ zD^M|dv1tAkN8S<~pkAEZIJr+D1CQd#*y-7HS;R7=ptH3=v3h9>sFmWz%Vf%F7Err1u$$c;;=-IPV2fdtjbtQ^!x>r@D6<4%d%AoVY* zAO{O+AEi4<@d|JcsuS$V7QE%dF)+(4kf;M=TzE6$LV&k*D-3Ob~ zW%{BP6-KkC&H3y%V|6ZU?rNVnfhYK|nXMg%-kUK8=Z3axsB77grXXaE4x%t2i^c1} zfcAC8e=hCwnu|C~PP#D&TX+*#Zn3j27S|ZF5*Uhl2M*-i+*j#`HYT-U+^QNz@QeQ| z>I;Vv{}FTMI4F<9CD}pwIQCa)`%K$feGY!AY>c)J^+O{X^EamOVW+(91vkXYI2h*d z>N{=1;(jL}ff?fKqMTAXg%t~mMpISMpy>hP|I_+y?={Ld%Anb>dwV>qYLFJ$p` z5SYs;g_N`5{7W{`&1-ndd1A8M?r{@5a442OBvNe4#pVuls{7M zP4^76=(eF6HMmDOdFCS<2eC3Jvft2i0m9C9;@^eWs`;X+roMn_uc2R&i;73DgyMhB zsl3?Zw_}RdLf;R^S?Wcr9gGQ<&Z-Vtcq|Wge+pNn*TMgo7u6RNtUqwqcRCJ79@|{3 zGGgg^k7IxvRwI55JMVLo8)!St4VWR|l=qoOtNW?6Jv z2gY#jNo&k5BhUN(s=jBJk=KDyoLr1+b8RWpxjZFW8A zCTP;{t-am(eYIvM$q8|Z(ZS!cdx;dr+MmE!3x0#-aM%tM@XO7eq)%zBQgj)dU)D74 z>p>P_^hsc>6~8fOO^|7w#tW*OZLMBqE8)5*XtAF))>UzThWp-4CnQrRdM>roew+gWH05O!klI5)|NGDj1feWXp}YUo65QpJeI*X zWH+hRNZN~8jNIT1sd(wzFg<)4b&Q~dw3}?_F9elGZXBuzk0=xmnD)8d{xqo*bSC*p&0b~Om`*HcH13s$kK)q(&ssL+ z=6wsrZC?4&bKCv#_=4Jt=eXyx)K}iEP~7j8qc8aPcHHckEX7J(hn2%k4!myO>Za*F zzvdajl%QKlLB|Y{`~kMPJTUf~AD7qMCn-eGw{l}I=iwQmaR(eOB*Tr)Egf^YMFQOk-X{5$-LY@oN1{{# z-UOkyELdCqyjfp1lW!yW#@gUF?qg{q@JKu$B)NFUS9TwAE%V*UIFrB-=gs7iSFG(s zaRc4rS#a}$zLeeH^NMS9>Wf=SR>v`%G#jOht=azU$}J+A&=pIkSj}(uLB81p#&8aU zj#oCt9_?1YmQb8goe+`xLgp2U`a%u%Ja_@!_a|;pGNpP7rl9tXHO6n;23X##a#EV` zDlTssJ?@2Z*gSw{lL1LOfP~tiIYN6bm77P$ofOU8DuAF zjeLOYq1mvI@=)Z0S&Hr}wq||uZGfE7X@eoKUS#ZrO_B7ahWF8*vf%Vp?^=AP_F~Ui zIK2v|)V#ty1bRnmIi)y(A;so?+~3HA*j*V7n$<45-KpfUZ%SYc=QpUGdx}v6?X(eM z?4bWSvU2RbwG1tRA^NJ^Zk6;vM=|J{4HfZ zq3t@4aRUz?QThXH9q|6_!)1iOTtwQjXaQwgjv&98~vtX4!t}arI zFWMCc5v4w?6L(=Wtn|ep?j-qT+|-w{>z$`MA;UF%I!GS@nT3!2O-`md_M+6Sz6dZ} zW70E!xGj29;njwL_WLPt(j7{K(@) z9b}hkU1Sf~lbu&(X*PmUmdL#05%EBQA!-zsnkl<(4&(7@wgkp-UJWI1*GOEH*L={R z+%CJ5dZbff6lZBEwej z?}_sZc{(lLGdGyZD1jl)oDnw2U1ijDi)!tnGP+i;C;ea%ZeuZvyiBoSO_X6xW9m0; zE-pOVsE|5E^@W+Imfek>O21vOFI$bzFoIJ5SJW53+DRt)+;irMf(BEV%JJzl!MP@gpzN-e z0V1@U_|~*Gxw;qgGsdG{Fl4ooFdXV6J3Cy%gkvK$G@cYDFox3)?EB_5TDr1KyMl@u z7&3MPROxuq@FuOl?to;?x&H8epWcA5$7hN_H}W-k!N|&bb1H<8*%z!+|->0p4dTSe#`}t2| z9d=_rC0<+RM`QYq61TKD*4p?Car#m2U)BTXmFrLE3eve#9%rH)D65dinZPJZOMaBw zIgc~-2`SD5#&G_bbnf!$II~f)bR`t7GA7hgJ+;(2zsZ;4t6fgD=azXztedd@Enm)- zx6ieV8)=)`T8qsy7B+RA()N#%a0pRu$+VAaZ*FViEC7Y8+Gm!E-A-;-h!r;x42OQN zluQ&>wg{vdMnP8Pk`pW@iWA9@B-m2hE9%9AHGE?NV>lNOXA&6MP!@SOx0?I@|%V~EL7{ke%jVpb;JQ&jD(Zc_!EPF&7 z!5G?OY3u^v*-_JTQSLb2x8wkHf}vLUuL(fe%X-*$bKP;kyZwB|by-ap`=Nnsh%D zXA$Nksv4#vZqq!K=%xQNL%T7X+_sX)Y;v2&7+d8wr7x@s(yWDt;0#>m)c67{hrO9rX%u5xO2|&;ip9 zrCHkGeyWRyk5f0QRYgsl>CIVz`Id_lJKET5tj|RFIu6{*2NDheCAspiA`*s&2N5RL zQjtE?PhhZB=-@nv{t`eAtqqcMG@i1}0qXW3eCoWOimq-_wR;huxVQqBgy6ay-dMz# zEMH_>`;*AG#95WwQ<_`CcRuXCttffAlJbiApSbnZoNPORnBKdMZj_kQV6vPK8(FL* zgZ9hM`92R#L8C#U=Io?LYq)sf+4gtResf;0g+`o67VuXa)%rpP%v4`}qpE3XAENvs z9#uLbm?FuQ-QI*j#Rzta5Az=OVmOITr#gWNZyq)O>IF~WwA0fUojYaKs;O}Ei507s zXAXK&!6g=_MFff+2d}+(&K*bA$kFP4hvL>7ZFcS9kM9yJf`@)lbRXMJEvV0?ndp9S z>J3?tV$n*uP9j)=t@~Z<{&kd6oC%U^i*T%Y*~d>d-*-Ao;@={Y41!N!3@7(JyB`V8 z;7xRAQ1P~H7vB1pU;POVmQ0)lKN)JA4ES|l^lw}XS@N(c6?apmU zlnq{!MV)O|mR|a3+7YCkz*tv;&d2#qn}XbP<^HXEp7f~S(Bm)m;|T~{NY;Yd8BHAy zn|exqe@EzH@~Jfy>?QWWVlUWRamtnhUfsE~U#)u^%wYEZU`5MKuYOQ)OGQ{AEV7Y& zY2$HIfBK1(e)eYS!=lXf2YWFx^`*x}w#e?%7)CAF0s^;%h&-ro;sB{aGp_G8OSRa>SX)Bs zq>JPu8|5($>wv7WTJQmaCbDm9c1JdwX=m`vGG=1$si4F*M>kTmvkr{moJ5=%s{)x? zVyAfS4J)o$`8ZhRR$@G_vUZEOg-@R}y6*`GLr5WWDUf#6C&!K1=b6KTCWVW1*1bZcd!v6`Y}8;P2K*rx)!^V5V_t1MW)_^OqC zPXBl9_k(?GNO-N5d22AIGY&XcI8*^D9ZO~xE-7<^*o38WNsCi9~N#&DL1Gg}G`bP7Dq U?#RR*^)ORe#Jo-5cir{>0JvlS4FCWD diff --git a/packages/pandas-gbq/prof/test_upload_chinese_unicode_data.prof b/packages/pandas-gbq/prof/test_upload_chinese_unicode_data.prof deleted file mode 100644 index 593d226a84c4df06c17059a7d789036b1cff7114..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 206840 zcmce92b`R>(Z0bxmvgtVu}w487_iTUUT!g%-i-;J_1@jy>({>9U9@|~=IDeLLJ6UV z5JJfp14)2{gb;cPf$$MRAcQ3ZQxd+V8ZZzHA^+#myprB`wYQx4bJ^d|vTk=IjYgx< zXfzsqI<^1mwN9(_!)NtoyLM(v3-fK0r?h2T^KIEFlk@G9r?f09WsAtREXuZ)ip^6S z!iTo**H0(Gs0};UzjJ&(Wi#EVZu_Pqidj``DxQ=nmYSv)mSuBGnwH?3FrOjaDyWb8rAh~v(2EjuotaiOp8s0Os*|coK`Gl zS~{{-V_FPL*=dDL>*7q1Ej9)Dd;pM4S7~7rdfib#uZdI#a40t>$iEhGTbH<9^!D91 zeEhqInr#NSE$TDe@&84%swn9G4mC`DEVpq0U0u=`k)Afnz z{MSaESlsIQW}BHYwZGCG0Z;c^y@{Pzi%>;R(`o_=gxi8EgZBqKa`2stc25Pz?6>s; z-(Pt_s#`Upo~Z*ghX|YeYT2!O=FZ(Z6`0KO{))pI&gV30OcyilS>T#!!&F)yU+E-x z>)?~FU;5#&m~yv)2<0IF7gh#U z?gtWTGb9l)hNNr;!%!70w~i%!z>J_VAL2kLp51(^nuOcXW2%7w3k!D7Sgs7^hOyUk z*=ymym4^RTYW~}~7u`-OR4tiecKdBPhw`YkJQ}^!>p{!*+Zq9++v}mxWJk8J9pUOA zeF(C-tjd(~3V&L~umY*uPwnsefq5zG#mp4vQEM;yv4(m@p`$AZmNm6zIyxea;wpBy zG2PkKQ40Arn3~E&l}2&fF;5>hqP>4A*;B{VnL@$#wIkG$!*(-VA9|Wjf(OoBanCou zvwzBF*v%b#(aq#=wdM=UI9N{;_Yvu~Y_`C!(rg)%f{L1&kHjiMHiJ2I7=EaH#*#zR zi%~M2=?Ir_g_+J`N3oa)@Ar-He!p2AfUf~U?L-|-5d700o0-~wb*2{`S;O&4L+OX| z)YO>?bLeq6mQ!s^rmYRCN+GXGP(WHs*e4`L&BF~u?gKr-;MS230<1F}kW20K|by!L{n_g(*DinNZP;2 zrF+dqN)f6-mWoXH7mnG`AX~z)sW8``2W=?6)nX)8Kd>2KIS4<5yHY0`0!d^fP(`Dd zVA2a&%W6j{cIbV{X5ebOtOAyoLLB%YT_RL8MT6(e@G%59>!8K&}!r}moUCei> zR;XO*C6Fn3g^e%AS6D2(p41$k&J+*F+MNpylqpYjUzJFjTAQGxgej_4vl9ziPV}>B zev5yk$PBhNq$3ZN3X59?BgchbE$yKtwFEzeZ*kU*!RQ)KYr66E8e0FRrq%h zL?^+`vuC`2*A`gLO-5~qB8lD8>)Q_f*)BIvNChS{HQIB%s`ndq<5c+zN~$xE1fN{@ ztK*vvS}SEUV6+{2!DvZQQ7m?#ii zT3b`L*86@Lgmwi~$XZu(vNfi|Tv$rCg{|Cnk%4>UVs##V4JS`Pso~kB(2-h8DBPA^ z+Pi|=!eW9IZ(+VO55^2nrpzNB#+mUBqetCAnb!xV12KRIA)vcwRW2hN(9N-Cmxofh zPq*kzRSoPCh7ENwsquXl8!5_*6b##!p{Ubf0Hr^cks)4n(Q#u@bVM4I1rYXl)u}7O64ZU-yAM$*Aq=);u&ntuknP z(o@8$KNcoTi}6J2IwB?O^^0b`ei7WDw!jaS-3UIokSQ%pqehgO!ZJ4?iR5loJx_QptU34)z;J!2B5E6mN1d6B97~%RsN)OsjJWt87mhRsU%e>qSP%l@l{*m zYw!*YRI-QnR#`PogH#q0ps|Zi7le3w5*VA>dJ*nC_kQpMWj3)i}jxQ zPHe?sxY=Xbp?R1QwK`a#MM>RC9BZwJonQUPL^#qA-rCfXRZU`F%&d~+7+YIR49VfSC3|G2IArlQWDn(i0e2}Dbtx&C?Z7|Gofr`iLc`hkXCmn)Lrwg+fK&!pt9C({GwLD%ae`X- zJF#|&w7LzD<>4wDKu63fEPeUKiMs7^g|Oa`s@?5myN@sABO{U5nb~MUJxD}7#fA2o z7W`7l5{&_ct|D}g*vjOg4dsaNY98T-rO{~3?a5=0zO(k`4IecJCWCU9vz-rhJ5W25 z7Tg=RSbfUnqhD_hOeQ&M&eqEKkcKYmtX~9Y*00DIgGD%Q-(Z4FLt)T(&b!MCz1F zBC@gNlOlHHwYfgV(VLE(J3f46cq%X%o&m1;ft8$+fu+OjqKp=@pYmv7I97Omm(G+* zJSeNzaIzcg?@9?!nil3JN5`7(m+9%ud9X2&0fROmrIVol74N+} zd-^sho570P2X>;!=4j?PXFUg_Jb2fROlM1524sneMf{>ac3B1EthucyvPm_?8|9|G zJPg!45b~nqU~I$C@B(}(Myjv>^2I-#ss^M2lK}-+;MdBh1bG}u0$dEW*r=Keh`0Ew zU;e8sB9WW~4a3K1ni$ZIEgmBsBDo0F$jU|9-Q#xfn1DvHC2Z<(78|0va%rue*T65iTI)h@pg+mU5?c#= zReTv5LdDg~MY7_vh|RjhW`y4SLkB39XbV8eG3*BuYk9L7uv}jZWy}hY=?L;FEG_KR z$FKjS(C!?b-{TTi6FY7eiDVvBFj|+%cr%D2-)iX;@6d zvII5%3kUZjj=?mrAU5jkp=PAk3=CplRP;-%in7ne>6G?4Al83_XG(efbV22pE`!TT zAp>>3Gt-(cE=?Dg6>;R<4m6y;>8)EJaUX2x>>0eQ)_N~~J_uDFO%g>*xN#APp;u#P z-WRND%mpA%POJrme4Zw?@If-cApD-kflq>R!kh`g(-+KXP$NQ}nH-GwdcEI?#9J2% zTzdv5Svc5Pmd?SNgtMKKHIKRx{x7)?7(5*bNxn&%e9?CxpA$kxJGW{&gy|4KtH;nY zv-~)4e5pV2WwH;>3oV~=^F%IHccap$H7ccy~d_sf6R2FK%@rskIf69Ud7qvP%o0qE*+~UOZVdwZFO% zWx@-9b9s=~-nogZjAm^c;mgQ=@Qm>qdIm>kQe-V+Jc{v8pk8d#B49XAYF6hn2Wlc{IjK*mw50jOnx<0_X_ZTY@4}_q!3NwM$ESe7>lQ zeTZb_@(6F;C1ESJ*sa=R!r65#$Oa8glazi}FgHPog0{}n-g=DNw?w7;XscRQC^hl0 z9^S)gIE)eBUl8Akf^M=mJ>eYVeymKlqkMP+Pw`;8q1%X_ZM)-JHH)*YmK6U9%iKoR z8?GDXki(0C8f5*cR_L>-R&|V^{c_23R}jmgZP{Wg{KLN3zXGCUJCu;l4bt*T8;f1H z&~^QZE!R@yk%%7fc5dr{&LVr0UKIQ*3l7FvS*G{+y~8q%OF}q4n2lL4S{g?IIaZhI zf)92_HpsLtOBYW}x(fB!!^>EFGnzBI=+i}^j?E;b8kU1JPc-7jW`~X^bHR0Yp%Lw& zqqap)z&F*qznJ!^?gTJEjxuGe^NGzsis$}YthFfLLdF`)5g3`qUs_o=II8KpwHJC}!Lm4d{Fz=oEKa_+>qhHfF{rn6iFmj`UPDtnL|d*G|cZt!GE5x_5Ki6zhZ_F+*C-0kI)9;9J3o zLhTJK(I2f7?8pA}h}5107lx^9&+Z@Kz!P4?(*4um;enk5rhZoo6gvpFBpKWa>5<$Z z_)AtXu9PpKtaeXj4LnH7eu0;Tso*hIFeY9>ONevZUY7nROApao?DRsKdSdbw{nwwE zAX@!jldoFna2k9_78^{PXh)W8>8+5-)kb^89#7sHXdG$kP5q=H)%lfzqoe&{ZF{P0 zwYc93E`4CXKi{zM^Jbf&qhcQji=;yYoH^k31UKhEyA1pbm_Q4L+fGLZPP6IaSVL4i z<&I($WdtTg8w#r9$>n13UfpanN!mCgvpwSS4BdyKo{&kt7Zr_sK<#iCvxr+b29%)v-&YQotU-wTYs!r0l~DsM>R#wl^UIBuLZ6( zsHK?X2Hb6AYJuye>e~d5bO8tMtprJ?cL#@HdRilUkOpqM>9yb83koC~b%%s7P4Rjl zZVFaUwaTy2kSc&!fjJwr8BVpMzItV)7%rteo=e#}n7b}wml#=@3Nayrk$n+(hR2fj zGpk3|Q(pl7M}pr$8~2zcG>-6?Gz*`rWi-=a3VoB^K8f9yk;55`$xMw(uVU$SBEzWc zdfn`IwmlEoX!ggZpf|OX)k*N&xY>`qaQ5Vs&2XUB{JaW1vwm6W zOpCU+@5t@dK)F?cAt8Z;N}j6$XiW1?~Ds(Y*?F0z1Yenx3nSF>O_|qG3CM zz0BY^2n{)3z&85}2pO>lZcNTrcL4=x;+AO-a+TMn)VEQo?*!GY&c%;F%Pk-HTYq8$ z+ZU{A4hTs0rec;}nR$K5H~a*cLete>0ka&i(TR_JPt*{hQJWg!H&DG!f+dTu8TRN0 zQ&ToW;k8>YR+*%at3IoQHl)u7C`rmAqPt)xDP9cXyy3BdPW(=&vw#pl<`aa84*~(F zWzgR!g9b#3aJBr2=`iR#NRv)#)L0z)P)DaBqLa7T;i}daW}0vbkY^#(DNbY; zCt~NDg~cbUWoStIk6X9v^brDh3-)m#D;cfNjrHb{)}u@(H56AE9HT$6`KqR}@CGcr zDNcKFq|VI*PTWt{*0OYfJyr>KNMOm0tY_8C6x)FMj={BtBt&DC&4EZH`YUHJe=EL? z9Q6jKYz8ApaP66(XpBdH=n*Myk?S`8b;RCpeb5}3i~+wJkY-_V3cR;qD-sPgIuK7G z_H}$&o|2&(`ruM^!uwVqT!2+77o+dk=<0#8>`{S(em})|WiMjcj9M&RbW9*a5)%Qc zqc%u8K-!3lMqzW(|?<5 zBL8moC6A}|_G07gGn9|ZmzmtgU)nlOC%I*3>-<@5CyedQrXQjZfF|2Ex7r=lT{{&o z&^?Sh=pOKDzZ#TIT&MJ4lBmqKs93XHFK~!$9b{zSV{9^XNxcSGRA%WxkGZ#EM4j?r zz-_rCUfWx|;_7y9P@~~ZtZ&!6zwr+z-QSoBOor~yec;3vts@AlR8n27r7j3hE;YNb z|MWNKjR5Xq-KIwPBeqN%n*9uA)T6@vj#eIPCuy~vrB%$3&>K?GZY#!H*#LfQJoi6t z?pBvnV102a6&az|eHQKyUf2PJFo&6%2n$9h=vc)0wtR0c1*Wh)l(p z08IAUaY%d%(wlBO%>LeQ_Dc^RGAtFCOt*Smx?Kk*iN;oz_@%sw$Az`GGuxS0%XpHB z#lX69$F_c+7Qq!5KHmMpp96#P0ErJ&uRs!yjVvB8n#cl#u}Cz&Bbked?QURV?r>(T zdHAgKN!iR|m2F2uWcYPZH7wS=!W415w)HUaE!N=pKa)eoGx-icF!oR5Kk@tbcunz! z4M1oWmrGoB_aVGHCFC?US8TuQVP z21ZYd=JF8jE+NJ|&*~QIV3KBz_v|84O{_XL4-#t*$WWoeg4;;?nz9M%5}N=*DvY4X zB$03`u)^G1(!-!Rj+6x`cJq{YXqKd*j)FV86!)bmxa$;3;xOG#LVe{pikKm@5tKbS zi!e6HIq8_l+c|L|)jQK&iO&dxC{Va^?6#$KiYBpgA}TBvyIKSV$f(*FZ3FVg-Lhf= zxLf@NsDL1KV3JL59wP|2%9zcH9mpqSlj9Fh1MBupydF%l)vax-=`=d5-Pf|kbXo?2 zjv$G)-SB#LQ1HZ;hzTH8bv2CqJldfEKlIkcRSe~|nC!_8-c6~rJ`qfFgU60Ui_7Fp zY4VJvGiS`)v3=VaEi)Jj@@fo~PJ;fMUAz0}3y;F(naW-vpvaf};P^)3?d0x#Iem8V*sSZCqakOu+=-0grbI$`*s zlNQZS1tx>qwZ-b20LK-%V#qUlqJ>tR#ZJaPCW}yv_XR2^5s?oxQk6l>e}uem;7bzN zNezj8A}j*Z#5p8<5gS3B1nDc6ESkQYmVSwlW9|67mkLLo zg&7>Vb-`~afcXukvryq+Nhd+mq4O53b>L1Zn*rT8N+Ba1l^b-hPn?v@#8;{xz6!QG zY1M$XQZ7Z5xCHF#)zHf~1Mfr`@IoKhj_q~?D=8E) z#Jxz{aiSfdJPP;v+3PfJ)?|vUVTkU#&4#xXPz=QY5IUes063ZGBU6#@<;aw}QBOOM z?lF6bF>Sh~!&`ts=jm*N^0?}H7QHYe41(uo5=Bbzw3V2F4jjJVtJkU+5D|x|JpQBp zv2O*c}Vq>oBUy084z9*j z^G{(%C)gN#nC|qtXY4SX;3W?(zhIqqRvx0$2-naaP>s!J2nq&I3=ibev5s70gQx=+GI$-;BCI3_u-go;8=U{={eFJ$bqIW8 zGDe`U#X%p2Vg1B;4r3=eFMFo{dt0$*41BwCgOUpyHaJYKDlT+6JqUzI<^2 zSDCTtut4!0x%q2N4Q%Eb7BL9hmdT(`lhN5QJrLdA7`X(vx1R?1rzGK95VqxkVPO`o zSfYe8dl*{9i8hBM)p01IF>@k~T5(Vnrc-=6OQ zO5aGrGR8FAh!748M}ly0Xob6d2U2~;QGKouBT8~vaO$b*SwzNG1sUg9v_7h*ONh@Y zkf7y(Km_6DFf7gn;T1_@;QTgSfb41!eH?f$WDd*~UHX&X+qDFfrX)_pHgjxInH4nzv8iWw{;WWzLp`yQOFAfmKt!&2DZ>``-!Z-skZdbt#2RF4MXRm7NR4;8Y zjv;mQ%-j@Gu{>0-JcpZAyiU0!Do*w?_C{^f#Mi(AA(P62z{cneMgO@`|6nkyra zA|X0nT1egTpPVg?LI~W*qIT~HDRV68Lc0P5!m~;L!GJH$5353-c-v$Us)c2qW0?^$ zX72HgmoFd3B81_WA{pvCF+F_$gj9u2U38g_*gAd`>QDcsRkOVtDnS7OhtlVHHB zE1sVDdYG~qgSz^=MxElRkXc5DF^rEJR$<}TVGvL!&xb#U#iw^W5p8PRieoi67V^ch zfN!Mo_^%cgTDCsPYWo9uEUrHt2eg<3PZs(&r@o1Ou*oPo)uUf-NMv!1`AbZVr1g8O zaS+L&1F(ln(NCQOhp*i-X8LQ`+nNkWfgk&Yo3Kbj3u4xQf13X_GdNW82$DfIrg=tX4)@puafzz?I~WDcQP6X5%|1D59JeH&% zYWy0pnMp9O48V%m1tib=W9iaAp81c#=$gp@SNii*{(sE)zz~3dfhbLO`R2?+} z6(m4B{MF{dkZz*YVwO3i(Hlf$xE%ULVLTIcOTnj1C!gZoQ3_tULKU4sIL7zQgS%){ z!m-%@=p+idg@r_-Z*7}WM^$6%cj$@gK6?i@SZw<;z((XDp*>3;Bh7}k!lN`g6iiU8 zmSTMuN>mn;-g22WcAMAhYf=!q9cTx{uX)0xS`xch-OHN}*k!w`umd!iZhtql)%O4F z`pw3GF&R~?9s#ctNvz!Rpn6n|Dq0ujJ>kgBZmjXQ8>?5ggYk>{3C6WNm{!2(KKud_ ziHVCDYAm)>zl@JW6xGToRNQ-EN!0p+%rt#Bgh{lki#cT``|L&{{+LYlNPLW5g-oLA zK;;IlVPG$=7-9vyQZ0F6Dx5!9()%4vWc3}ZLYF#qS=CoG6mN&?#E(Qx_md5FmGkJy zIQGM>E!b?Qr(5#*5;~YEG>4aQpa!A|nEB8Jw_JFf0_m;SyH{bET=ozcbo)V`HY9oq(u#4(5kes}p)ftdMX);@uCi;$4gGlqXW* z(WV|og3vxL&aU8E_O}Y2w(c`dw0O8VRsKUXAMXyYuYaO5%4Jw)+TVV3|3nZVr4K(`@m5@ z_G|>Qn5_Cr;`8L9xDx5jha~H1Rdq8OImhZd99oru0o^(h3jK5L{qsK3PdUhMn6CaBSH*4WC ziVD1mQdqLRHc+6G&Kl`41Zdjv)gO)~W{}&9@x@Rpw%E+bRtd=E1~+f82u zBqPyx!Yh<&5j|b&6obUo3d4^n7{(Q#Q$wZd$P^H1h)=`iF|LUUyj4iOEhDf$(Pv`g zkJ^m!-*V&VSvi3Q*NMsvsGvN8dL3-Os#V{*MEj1f%~?ZWGMK}ACgyOjUgx4;OR-6N zw?q$hoe#-eo~*x-E2wyflG;CxmLtC^dA}=0Oqr>%)uV6ZiHYhcvG{C=P#W@&Be@!I zMY@B5ea^vO62wM`le~=C2fC2K%!FRlN&)2@Lrs*2=|2&W8kftsxpQ3Tt@~83R8M)L=$QqJf>C(fwK1|Yfspu}Txqd8zopV-c~9^X=*OP7aTwo>#2*7<6beVz(l5?S1paRd^psJIv^=E%`Q1O}C_a?$j8PpMYK!%F*%&@4X;5YwWNX+bv3N zhsG)|liWulS7Q`mmS~8cqXf8)d2}i$Z@{Z-2e^^Vy&v@4GCuFI6|pFz?#MO{#X?#h ziuKDo95F~mi^Rvx>Q5+&4mc5D`)jVRf_3y(X#`ovQkeH&5)`&on@qANoAx|`7>AU5 z8uD9OQQAbp zZ}WIKFx1P4GeTMpG#Mb4PAD6w2**{%FSFP*d;0d%XTkkw3Eq3x!9J$J0}&k6lyp|b zBWu}d1?|?!cztj=wgaxO&XtE~k7LT`8gn{kB1sZ{lb#8WC72KTfb!wnm_9lQo*yvb z%G67E^yPtZ7YteSue~u*-w-hZ!=-p^`&axFub{y0r6(%i+_3)QHiP-m+zT=#X~Lxd zwsTsDp+Wo(jhIt6r~R4Du-z4-r9SkImXOCp=UOt$fc9!d2H)zBoh*>CZ;`wga{(Cx zdx%NLSFEg$$&(pPUSo7Aa3(9{3`sN#^ zYz8B&FLI4#*ZRX0m|!O}Kz5`IINb-U4i212Mo0jy!(=eN=b@^Qw=+)y2%X2f6bOBmj4-lW~-adrYkjpm2X8h^xL}1Q-#-@`sXcQy_bY znB5%o>w`x~5|+uxsty8H4#Gf|N8<)DxD|IZ=_?pTwQ?w$Xd5={1{n5wuxW)Vd3jt! zeK~*A{(cyo1a;)hF?Kqp1`7eNBO5WX52x7-+W0Zr5Fs@VBFEWG(`^VXUeJE~b@@=u zLCuw2*~W)5tK4CXckGAu2p$C+kP}6`jRBn`FwsNocj z_P<%f2(ez6gRaCdo6VXH4?h(&Mai(xfRaOHp-E#CP)i$0%0q5$!+TK{!pF9#FL9y8 z^*N$Uth|nP;;%IKS}DfGR-+WR0lSb%57)0TUdSH2$Z0$Z*J1I} zg1ib%A8n|@zacJL;n(1yZ;zjT+a0(Mo4Tm`gY~x9Ii*fUClUlwhfUk$8;=c51ttRm zy@(3J8SLp$F!@5%icoWOc&)$mY0XK;HNZXwhAi-dz9}qIuWs4CxT9k(6&4qVg@VXt z^*%c_2zn5L=+qLfu@P6Vf(q#K0#cJnw4Z?W$e5A9g)8tw_!vi`p}j+%#3ig0kTpCO zXxbDwfy~!PgU+7V<7!;2ISeDsm_ zedvH<^ey!FR-(_ux<=THnvAaj;uI$T&}I`25|Yn!5}mBUsb9p~raU_A zT$1k{NP6p4RsV)Q)LD!FYX}OST=I?p)ex|{n2yk^9E&ehGv}~1o{RZ$0O+T*C!Drw z!xU503XnVwU(B?Vt?Qf@qcqj#ie7di&qk+pdK)73zHr5S;=`ntV}mH-yYPO{aCpGr zaz5i@BKkj2skRCwlTThR%sjQVnf#5d7ax~1$D6RYss4CwSZfBld} z$HNk!umQ#5#K(w>^rgTn#ps?>w~XO68ZcU5`+OBXDHG~z;k@7lmlXb~U(gEkX}k$5 z$CXYUF{V^&Q&qo0577yyR>^#hc{r(OATbjlo}2-tv>o&xl&ypHUJhDiqKs!{Rt&$9 z05J#f_9PzF&H;L1g?$@{K>zaEh{M%Wm!?J~v_Tg3iUM8yprkGA$}PtEA#H0^OpfBP zGKt1|rP=PSZ1!D>d+M#CGbNO0x!Ok!Q^L49eyDW`BZ~d|Mo+LYO&@(%SwEu^PqJuh#bY{E?Bg*qcSfQUs)$lB0yjGn=x;Wg1 zL@6JeQip1g*t=JP4km}ms8dlKIO{k?OrnFSxUi)f0C{i>{!@Wl{9`-uWVbf2(=E7L zz}OF-VfPN34%Fu%3AX*B3Bnf7(304xvYTeZQYEianD{xt&5!(bk4w+qBc*1_B*?KA zCbd+d)jRe=AS6#3vT_0xt_MQF^qIzcXvuhRRs%E{^(b0Lm+Po`>JvR;F;(4@UW#71 zsKqTJwPZ&Vu&Nx4_9FecGo$odt8OL1Z~EgneYsTZ{P}#fa=U5}X?ulPi&yXf`=6*c zq_tj0H{_#ZOG1RJP2e0g?&g^(*Mt`2$}k@Hh2qx&J6)ca%be2Zkp#@eotD~zLop2M zcqe>05I94>pE`oSFtKZ_&8UoY$ANguYcD()4d#Ewo=jzcLZYL!1T<~m5;b|0vxk~% zoZ-=^LMM@D=PbxH?dv%!g^upc?(oJx9*2`h-L!#8u1BRK?;;*F>C+}_yRjKmF$lj_ zI+0*qY0YOD&PKvDhnuo#Ceg))Elh;1EmG=p;mloFH4{rkn*kwMbvOg&ELf)E{Li-K_Aq#60lerhqp#>O{1U z;j6PQtCd2BJ@B8rVCX5dpo2m^p}u(*EqTwdcTB9Sg3TE4ud~KqOYQRs&cxZJq(9F$ zt2gmAaJ+>eOst2T&6uEwb$V(V<*S5HP_@V8!TFw|y2tD*rr`QKnUJt4`ymz|5W0>C zF{!aacHwJ)xU~v@>yMQ_*mf=Mg$!Opr3^9}+NXZTn9~G*nmYuv(v=%jW=WQ~JGK#? zh@Fu@TU_%vQj}dfu$np4r`DUn*p>&RTRW4dlRl$v0@45}Czfh$XS41jq2{P(ZJupi#%%zzT- z!65A^z%i#Zp7GwW{b?Ivx$q)Vh&pl+XD?Ko>+=kndzzxOg~is@+O}9?qR>e2#L>Hb zH@ceVki^(ds@R({YrW3QvKHDv5Zjlw0v@xJ5TzjMfSZ)oekLl>2zEeY@;|I*yi`Ls zuy`%#i?hIaqQ~@QI}A*(@3x@X&p-gBLo5!7&WUwQ%<8Fc?Kdm<;yB;*s?dcGPf>x` zdiDc7zECdDlpDz#y(cz`9n^(%OCwMS5B_EY+L!eanPXKe4UJ*il;pjQ@^F+iR4tv1 zCb<#Si4}*z{&(zn-6ziuGfjqV04a}*6^w?XM-!D+LWpfRgqtU}iOYf=#z+?n$Y!U9 zVSKd%iUBP*CF~AerfG!=j;{M{);-c0CnM)R{kE=7tZCk6BFcw(%qF;=Y1bZ$)m+d= zT$zM7?1(+ptajE|Rni2x9r+q2kWWD+AnsVbio^8f=v_#ZZAkLnq@UGnl8}2!gP%k4 zkHhwp#&_7#w7MZCxq9tAL_+Bbeh_P~#2zrQ20NPpacCb^qw%XAZarq&9;tOl=TK!< zLc9N0bfpC{=dgArwo;o>a|DDsh$tdf)iX>0eD%)xWp;P~!?P#)hidgFE;=;)17HNR zZaXBa$gjzv!T-U*AE?iklTH9$18Psf+07B#bCUti$6(S{c0^$lMb|0HwQwM~z94{u z46KN0-{;uFBWfgM)Hc)6I*q8c1#QRB23xDlsQv;{S04};jy(svMyr`S#;w5dJRlxE zHU!{rA@_x2r@=31#9+jfa6A#v)ydsPorz(bK3x1xD1i1`PW*xlJqZE=LCE!jr+-ZAINp(+%MDyD z=DHYPR$f6;9*oxw`PellhFgq0_5pcWCJdnil^gYYO)V$|&!Y;oQtTo?p=jnOs3N7F z8i`T^VP*lkiP4bhQlrFB$Rwlv@Qq!0Xji~v+K)l2KbHCOBq3#iwX-1@p0nEj)QCEdv#5Tz8IRlBf^I0RbTITaZT^h~;dY1v zeT5gJ5Z2HxQjT>!VyVkOdSF&(=AeN_42}APa{h4vAI~qCL<5$jC&9zfAbBLMjMI?o zg&+^VUr||JX;yoo4DfK`$9B|Y)DQt#KxSd$?@F;9={wgW9CGP9l}xxXpnxwA;^7p& zsDb)Z#=P7}t!^AB7cIK$H1Xq#p;) z;Uq+f4V@Dn~7yun?c=Iu!9TmE6%9sMTO^KxP@rv^>0_w!%ETQ0uYC|Xg4t< zH7eJGGL*bFTpr|!jzv+hmF+UImQ|baPaEy)Dq=?8G-X1xIMVueOQQqHTiJakHZHr> zj)CD8_#xbk7}{&w4oBBHEAmWcYE+bsVBg6nW_VsA;XqtDBO)xt6_x_h&5fIncy#3H z`X)P^{w_v_Iev5?vR_{-6}d01l~S}4Ck9B43^zyx@9cJ*`rFjqQfdPMaa#|MKS7~u zNWZI?MAN|a&f>M>*SFfP`-aU^(KAiq2^d$MM5Nw26|`-<^MH#7?94qD4>v=eVnWH+ zk5EM3ciPV|Eb(P*-RL_2@o#v8@&iG1&79>?VAUL+wuDQW*&tNzBCac0)_d4W&66HG zo{;g38wVc;b_xD8fwh8Nr=<#4OxZo9xW7n{)}x!6AhkGJdDBB}BDgpZbGT@C8KBiE zXe!ipy2$Is4Y}Phn+zzk4*RZ8oCkw4+FA=ltM>>og+uYNNr1R&J$?ih4-yMT>ME}r zdp3{PdZH`Olon2d6lsBAfX0sxnB7Rs{>~&aINriM6YFtjGZA>GFZT|y{-im zD_f?}G~fy|GcN?=fFzgCB~3b>m_)f>qln^_kn(!yQg&%;w!k9*y}za^jkjXNQNbxA zKtX@(Duv~{^wnDvt8c1`(?kd2b9BHfK)*Z8({;YAfxFKjQ1z0w(k$dF>1NUg3P$2# zQyU9bd&3xk;o|mMacIQVa+s4a4Rka<1D*q6WxWzUiruinSp5*chQC6Y@?eZiskCfA zE`!w^9^1X330-F022}gG%sHnqdQGVRK?A7q7etFm#Az`?!amyvPR`K@wy4cb`BoSL z_4s0YX+w}W4b$2ZOa`QB1SeGH1H9Y_0~D2MYyi*|S4x;aHvK^yeYy7xmKw@yz0RiV z6n?K58v6;+8pkAB7Mz{KPJfdz;0z#`v^2VbsMQ7?Y1LoQt7M^SL6c;VW)un;4kcNr zPH)!WsNB5YM7I`tG*{jO1zbS4%Rd^JzWTtu+)?=~_8=y&aDFr*`@?T6U$tjST`j!m zJS#k2#OToQr*=i64OH@WpWPePnJH1xa4Hbc(S3{!bo*^<6gOH;v@3bWnjA|kz7DvF z+H704qm+r2ujZ@dq!V)}F(2i)(Y9Szl8U*L6~M+0YjGIlup^8Z+E!6#L{0BQRxuG( zbmhLV5)7pN#3Z`l@hk*KnFi=~#GfhI=ABgY#!b;EkBR3%gu7z|G6qD~39o%)Ky)D9 z96>)XM!9f(nyjLEScfUpqk`8!awQY1mfDQE2=yfds^7b4FI(ME>@CjZjH=@zM#Y(P z=~=WYm#}S!9HXvws}q4KP^y*{acbU)`3?}b6ZJ?nHS*Z{J|NATrViZiBy}#-NH28> z(Bf-~-xg}tW#v9}T-9fc=Eazqz{PaS5z6ZpsGtoT0runSJn5pNf+gB_hh2Qxk*`mn zvc3fg@Gyrb;xE@-3vM#WVXt$08{Uj`Ho9a4c(;rs()n9AXkn;+K8BAqp4^`-*<ev7Ck>ofegq~M)$*cPKV$^eOr`n ze?M}TP|tA>VD-mJM03jTIK~43gDHF&&Y;kwhIVN51jKdtR?h0B^D$|mzAh7K4Q02WwwZcMEd8X!)37sO3DV(e=f!sNFZ1;66@93tphBX&v0JgO2bfs0rY2{9S4k zGv%RY_d#^7m|sEUQ=mhCklB8W&cb5bn#oz`nhv@coI{c`slhU*;A?=m$&A1C#|?HP z{)RK-eHcld1pR-y$G6^j9ifU%#t)NbUJJ3d9MDj^4g3Y;CxIQYjOWp^8%cj?Hk6jv z2i-#^kriP*bN6UH3Z}CFBpZQ2bVkqt#AA0-WDsNFYfX`tNUb3%+q#4r1yHFRf|}Gc zNJRVGgUzUo&^)kp<{G<&F&VWx?*R zJ&(QDt5}4DtI=ww(!Z57No(-Wus6bOhUg-RHXmV3759ve>6SU&`!>ZNlbY6OeL4FetbTaL z3I2_xtiEKD!*xo1Hy6lP(>KNk%SBIAXK;|Cb(|i3a@)%+HFoga zRACa`zCegQ6Cl?V`iMtl19EL430i+&t{YiR_`#%AbVwQifNf8Z%!U zs|tr=;T7@oCO1+Vg1}o^ZBYD1BMgtTvVP&{>pg*|qg9 zWZod4G_;7zSG4hV@r~ZtLSi*kqg0qfRHsl2t?>?yG(I4uE(emRB?n504N#X|%cknb_`61TV`4<@CHU9jnqJ#}Ix|aQ`>F6AoNC1!ir)7cZ65j> ztpit3V1>^jF)NM}`evbtgg#arPv~1CqRIG@LWD^&{MVf zU`n23Run2IuZP#J(1%l94Ogv*OPx{W&qx~j1pQjw^HDNTj>dCW6>!gBtd#-oJ z@beGljU)}u4Z9+T096NW41M-YMjZguYxl-Dj)QKra#l_TBz7)98Su`)W7aIYc=dYZ z3;C1Sc{hX4Zh!9naTna53QVROXNkvs^=kwpGcL~Z1q2D(+t30%kLCm?(i*gcJ;1!j za{x>#tB>y+OlYVkDs*Hl55*-5wXDFMG0E%@O07G5;wAGy8hsikK%m7RHg z-<#H+^1?&YQh~{It1e)S7VE$yDOu-`0D*xxdwT5diso*$r--ULFv;e8QB_=4oKAne zcEn}S0dyCDK;oh)+Q^_(wAmY^Mb!%YT=_0xc)bK&UBEmO3uZIPwzt_;-XNJgCDW?k z36OaD>C!?5oS05`VtmuG2$mfk58LfX#3cuDyG({+W6j@Em4?KJxtRCJSxaKhlZk|2k{kyin%~z8lCM$qMI0Z{$c~Dd#hKQbBI*A^v z#5?hV);Nf7^+y)jo(AO6nKniMX9$|;^;#5m+cDW$$6nAdg_qH)I}!_EJcbR5y7kpw zq%#>)^iCEX*9z}#1rMqMp208}a98iz{({xikx^w)S23I9ta@Y=V?gX(XG_NEnv79} z2{L+=Q0Iw6aR|#o`KB}JsS5RMz$iaqU`Sn|OOf-3VpYf8y zV#-5zOTmlM#YK*`4T-q&ICAK0^R|6M?&HHeSb*DEslq4a!_xas580 zH9yC))?3tM%!CV{1hvW|TDugV(~D{W<_m}x6RUaIj5#=UAYLqoB)02&1lBK0Plu^Z zqv>{LHy$_X!Ik^BKyHf!Wjt=7B@YN};2Pu%+LF`WVb2flfa@!G8<6#1MM$$)`qQVL46&@3OcKGKXw#0%xg{O9 z=`@m@Nwk?lXuc26I;8miCYwTkrE3+MbHP-+_dnEP0a<{4#1*91W7hMp3(^k6T69gu zgk-s%wJ@Ezd8<0Rc^wsjEj;zQG1FwrjntGnQxMmaQY06C0e6r(9%V0S`MGL zSNc$>^I4UPt7hTh;0v7u`|LPv>7R~Vkg^$$+${8}vYgm7=o&#Mw^+zap5slNi}5Mw za=Y|m=5{y#=P z|LW;>Gd1;naNZY$QI(o@ZvDY&{zT}+$8?_^ZX%%-nJ9aN72SaGMOZbZYu5O>S}bM}6QyhpPwQ6Cweu2!$$C4vNHq+mo$DZe?P3={92tMGLMr zGp+(rz!NJk4fyd@KiHwUEoVsoZeMxPw$u(x4NWSVb2Cl21?rMo!3~3M(yCNgnF=X z*mI;ewcGeINz7+Bx-Nx65s?-*x;wCk#|4t&^fpEni7y>GaE4BTna7|1>_!>v>P?1< zW(#L|Cs+WHZV)BdyuSP(zh+{AZ6=A*Op8(>U+BoTN00W^#VOQLVwnWb4*vIn_vbdo z=5b)%n&BJn)HO08>cI5Ay?J}> zQ%mcM9d&f8tE9bZ%`gU&Y;Qe}f9WhgoeA3-#-U>piAxYy1n9=lqF%-KkOvD0!z5ZN z8_k;82<<@ZmOli8;#w0Z$1CrsoBM-0II1Pm4JQgcEy&;N8l@{bK7GI0-|Kp-w7=IC zITb(3jc&s^f#pe!BuV}Kg(j&!#z>A|^A&%-{<7kSsla5q)g!WAIogA8+{M~3cv?Roat2|;sAVqL!2m<5Ur}1QWCVF!(%umnb`JrSN_C55o3B~8Ir(f{ImA8MR(3+ywppOGi=d{|077GMt zSQ0WYU8{J_Nb}{)u&n+ts}ul3o;N-?{~-^V)Akt$5^UnOd=&>vy{zrEBZ^K14~QAj3h%zXtVhN7zQ`zWDiNdu0(yxXD4@z`cp z^J3KGedo_Vbe?% zL1rH8{dC>vEjq@=8yzf6!NT!uH3}X~5{0;3HPfcmiu$cPjYHq<9=C5QFqv-k5J&+o za+hOX|MgD?Ok8WPRA4fo7gd?c-bnnx^By8eB0I%S5T1-UJoknfZNEeCx0@brd6P}% z>gH$(U9Igm+R{Y7C&j%iP_6@?#9>Q|!%6VGj15%{B;0J0BcmP6kJs5RdwO#}c)t?^DL>+EP zW~_x0mzzJyk4%_Ef|K$gF-oL#*J=Fr8@gMl24n;Cuka>Z-PPrsf&bm=G96jQs zuaL++l2bQ+?Y{q9vS%u|vUK^`&kxuaMDg5eJeXwrv+Xot`@BKYE4sL;Ak!a4&?jF$ z=Iu<=I>2E{HA{Lh^>Cmt?+p&&Y&4^j;PG>}yJD|QkP4P zowxkGyd9s09cYA|lic)rzg}{qWiiaUp2$yY~6v#8)Qlu6CDNHpd9- zSUACn`O|(q_w2c;;Mza_@`VqdZcAZdm;<127gWZlwlg>;ly?`v1&%d68&i@3&Sab9 zg84YR-0zSh5exz05fU@Q$w7(sxIP?BZhPYLzr3`G)--MfIw2Lg@&MfO<#D)^GpB#= z#Fuy97Z%&K&i~cP<0fv9QcuW$xv6XyA7hSBqU$uiB)W&|cSfoNoLhOtFtDYu5Nt_I zzr^LlY-c9yz`L#rDlWGWeC$}=C0(e~Sh5Sg0^xYBW;d>6D@<&3b(`r{(=?yxpL>Am zsdayGs@b~hmy8J93r(hm`eFir+DM@4uRvl?-OcFsi!pT*;7IKw#^j^J>7NTn+ zSUVfEJJxy-1`i2o?j;P9$Xb97$eb9RGRV2@v~V&WPBWP#a?bu&rOVX9F6OH@`}~n_ z{i|t4Dli$U!7G4M_zZU?Y2+|b+zGEdga9`ulQEs)IjBCA4wfvwX4s=2OikGgF})B! zhnHdUsc#~O&8IuW61w2@-5ex#n``lPU~HW)63waH!yxW5>t_;t?Y+JK`K=VDm7Hen zX?KJA8hUv`nr}3wC9nw|unyB0BN|R)Rt4+O)9Bn>EssD2^3LE zo^qGVOxgQb!Q(ixTRk21#PCq^P&0E3W6JK6jAcB_P#p+rv72=}Wgg3KK?M zDf7~SqqzRq-WXK5NyBqM5V-n{Ay>lUVES)LC(D3?YrqoTG++WJ zx=Jk$#^_|uI-DT3XC3A&8tsnd++IQ692C(S4LhzwC-f)ck;j^Sla1*_1X5{M_fQnY z6;s==SSZ$~DpLTL7Z$e`XJWuJ=qg0B6Z$dzkmRq3S%0>nQJu^N{;0X10!jO+KLE1?bv$@mzdMzQ*Kd}BXhB>{?<`#XmFoIEc-H?WyUHkW zxMm4oWEfNt{9VG)r4gFqETF2BV6(<)Z+?C!93_K2E^$LxWftADWc1~;7}coaF*UkBdSug^iBC455+ zUIrV!ht>;BjS-w(ogR#>Q(K(0Yi7JoQ&k|e!3`3>P7lV`c`N%BO%mr^CyVaK##iNI zI2*qqeXj#!>!jiaB>=&^c=JkSJM*~USWFP2MgzQInINtQV;dMtnXCt+K|6rxBzk`s zmnCFBvK0ssAw3IK_W)z-P_8rWJV!MQ(tvc=2Vcn_VWfr-Hv<@J9_ql@I`uYUYjho>`b~~1lVHMI=XPE=Yccjm_dd7ZsYjoz1#}e`42W^T z#^3t+dsDUo{In&X-ahN_lT+#jKqrKzQBdp(X;Jt;b_*C2g~dQcC&BYw`Mp}sT7(D7 zo2E>M5JOKf|&xNzk z-D%EZeTHNoPE(7>Sq(AwyR&ZppRc8l-yF-rU;eYx6Ia3V3|_en9aDFLVakId|F8tF zn+;{OHQqR-IGX{o03DPAB@+oMkZ@en92y^@TGb(FIQYx|diVV=UIydJ;cK^ynf}^L zP~ufmD_*AuL+a|T{e*S$`3H2()%l}f@T1#wZ(^>D&8k(W2V?8J0nMw~UOc7QPvdqu1hy>xUdP&tax#^VOVsaj9cl zHG;U?Y)_rk@nAUE)_q!CD|Nap!}^=e{p76iI9q)7joWTp`|cglTMthU#@2ZrNAi3b zNyn7dylptI+xK_%U|1&?6a9uOHin{a%C#VB2W}%II&18*Kea+B3Vw!akv5?y7vh>n zmA|W%$|e()4M(!=$yzeS_GNa{WikeQ`kI03*T;HayVm8t7~Ii}6IsK@FNH}L5w*)O zh1C`Y{+=rz5S8r#%3B$myun zc*bCCosRwuv8Ue+8KceHYp%{6IS-gvQd<~Xr&E#|!eQVZodlh)-8=Okckhu3PT2Rn z^JbiO4Cdd7ve39Ovi3CN8U!X>f)NjCtw!7uZF6!i89G)y1dZFJ8jFEFS3w2$5#>o} zDta~vfs=T>4!2WK)U|E-!)!d3M@6YMM{UD%VhOL|;99MIS{N17AhMZoYOqW)H=%`J zV8$^XscM3NIDu=XUp#nF1E=5>obnSQZy%cY^rIdHzAy!J65`gnMVKp@CRDtWhI{^Y z$J56)!k+v=*Tm2NJbbg1dI^mrYO%ij8_qmA_x-IQ`j>)|2@ul_>xT$(&(Y&Wnhn8l zndc>AXL}9W%01K^N{&aomg;(q$s;llPfV#>1jMa6_B>|S>#yACp2ugW)FkPE12HC= z=t|x8dezZt{MBR$t(P}czCnqyC59m=o$|9CTd&3@{O>1zM#dzxvt9VOE!v7XJkKuF?~-anA0Nc{4jTqaF*0}5zDK?lf)yIFCZ~S zMnV5$0px4UCnII8fVr7uXkfTDu9xRx+;tMn?0$Ypc@nH5kKAzJO^^MSOZ-Jb0mt}a zg&Q0@NQnkeOIIe*<8$zz5`s?QhD(!QE7A6e2*Evhu~DWz0!o-8!00iA82ek^Hj1~3 z9vi5}@n5bM+w*(LxRIH{^?UCtpSbhB+wfGf$rwE$B&F1Jkui1PZT$?iD#145{uu*W1>@gzUH7-5oz_!lTMERwhfvo!nyCFk>U1ln^d3$OpLHH`>0jR&p}X}@fQC!Zi}H|S zrX2w+aibos)G1|K(dSBo82g?#CF6RNZAmCU0N9@?*gL(+gR%9_CRw&cf;w~T2^0HZ zdtgXu>#PG~>I^r;=;1+ehwSe-oJ=B{X~M(%hr&UQ)OHy$8TE4kadc|X%Vg}zu^9}K zNo^}!?z(TZmgt1&-`cSA7%CNiipOQ}oTg-bE6jtL-D-r?SqFyQ&tqbR+^wpaNI+7? z=GRGZW&hot`FRF*uu11W9=VPu0T6kOkOt6evEjYXS zWk?4YCuoX;e+kZYU|6S)8Kh@ww1&DFDItlPV)i8ie;TGvd7wPD0#es0Vw-SLj2dF_ zH%$D^pe3J8G1s>d-(0;ev)-XC+5y;3J@WNG-QICXN*yU6PB6#_`fQZ!-E#BIhIXbOz{glnvh*V%U0yXfqY$wVZ zE=_uP%P%WpXV}dma2#eL!*SkXgp$RKgP?XJe4L_2blZoeYG5YJh08ReyiBQqnO!=J z0Z)0Di8RKhMPR2;9IS5b#r=mhd^tQ_ptoxYZ-?jF!vKz2;ap4^#5cxXir?`^$f~Xm zjA>0PfZ>jA^*pC#%?+x7A$1SqtSvKn0SW4)^b7Ao*DIypZg2QnfOmJu^bxo-{Z}f|1ft>N|9!ujjKh-jK}bHJ!7xa{yl0J zy@Y6b-wRNf@7fTMP2-&@cH3mod-~LIN{^=L`y?? zs%<+AsMHy=IeIX*PA$86RVHE*9SSvg?&FsyZ~Qo%-LJU&p!e?i;n!T9X8=^zQV+(~ z`8DEsq{g!{m-)S@BiL15Bpf3RM0oP&9$;7}S``0+)qY?DuBUiNdU^q2b$RRLR-(rw zinjm>VZzyTtL=%>fO;@!ZuN;@{roR1YeK;1TpOaL!H)!};{DcQ6a3E#izy8%=K!&C zV;ZrO7sIEiq|gyfHzwBA!e+X$G+3IbEoF^#<|rcvY~T#v7pI|G3niIZXE>K?P!Tzx zP|fpLoJ>L710ak)FJdsP^F__G0u_*Jxi zXP0JM(Mp|CwiT9A2gq9D;PIvY1hy4b63pwsm^#B5q{h}Ha2#`6r?KjF*QwLM*gB78 zosp95aIk~6bl4^T-iFk`Aio9B@LO0T%Y$%lgNsCh7G4>)aUighcS#~;$f+2D3Jg#t z)9pu4H0VY!cjyg=u9J^|l?UMkJzgVB(8DW3+?xQd@5YwL$l`}JkQd1CmW6)7!gbv% zZo4rHBPA}0v^jN@zM$1L=uPDokSP%~IL_|YXlyc;P<4oDu|9|+a4?btx^~%k?V~?T z(d@4NMwmErdoZk1M^K~2TyD@IvQX>GY=KGe=)Lz`G;k#X!I%t5uY-j>2evW(UW|k+ z;gBW75Td=93>vtGwJm0%Ic9)bnr)c#X3LtqxyBo`~ro^ZQW>_`foI11KQyzpoS^q8Dd zM+=B+)@)mC#1h-=nH;#YNVPZ>jFKMw?Ss$4n|1~jBG%CF5tRqxLNhkJz`|kb6){c7 z6Gq1nrFat`pSbNlUpo!4YsL697wfavv#sXuu%b@Ra5pKSvzAmoy6@IACNA``Fa&n0Cc^Iegz9F6oF+iJ8E<)zMsH1ezo-SU|8od;H(N?I*PpW#5!Ce zzu5y6lR-ossk3@Pg(1PfL{+L*K$@3@wMC3&BK>5gQD#? z1}ZrIK2VUT17kWCevP$4%(i85!W^#_P3n)mV_L(P&VymK)b1)5;3st`(TLY+EheA- z?{7*QQycVk8xO{!I+mjASX7KUtLYop{+3I)zRH~dEw2+rA%=cdD0Sf#`sm%HIu+Ok z^K7sT0wp>jdL53%jUk2-&5n?2t4;eDydjdhmPC3Rjnt_B4#NR==v(0=s%j_4Xrjio ztD1V|2*6i3bmg&;ni^@^qNm0OMc4-JC_6QdYM9MIA%d(xjbt(kNdiHupZ~qz?3W%s zP(8*Z+Jm@TG4_DMs9U($&35Af5Iq=M=Wj`jU5G}Ww-8`JC|4b@j=>O_Q;Vo{dPleuipjepr<;zfreP=QW@%sIz4Uv}BSFuO$=PO3Sq zek&y9Mpr9rN_C1Qv2uD#>^^!OgkraO#j*r7!DKY?flD|ZNW&3M*9fB+19DMoaaM6t z!WK;;wVAihYK&YD5i5dAtgBGQbE1jGK2a;lnm7Z2i`m}Pfd>Ms+xo`*!^oSk|8^qw z5beJ-q|)LeAvyJlkdw5ap8mKcCC)?YW|=T`;7LerJh&L3>YZxAWa~E)p<6Agp4c9Y zt&;``b&yQLYGDECeR03s_$Iw%byn9Y&Bb6?=V}fOZ_Jj*Y0r{uI_Su^;AM+wA*wC% z4YObuT;^*9+G9L3N!L2g%Tt@8s$_ zJs4Z3URqZ5Sf@dzTfIw|n(Li`v2{9H#T5hrLu@ZW2F?V_W0Fhl2ac#wKSclMC!f>s zasTP51o&cM(i#k6Rzr)!4>pYH}`k@R3orbjgeP9g;#s$d6NO8xv7EhxJ`OV6Dh!{7fKHDsf z^X1V=^^CE!d}2X2k(NFIk9qtcKgD2d3))?DTD5v1n+qG)d{r}-8arFnvT7do;BEci zWe*w|9|4ih5D+YZy48BbYJx}zKI$Hft@=+K2OR(?&dTEHipS$@*nRbpOb;hCJs4Z( z%dAs}ua4JgyJtmDM2?UHq)rubxcWVKTmL@@uj5_E>$k%roqZcaxDAIS?q%b_*g8L~ zNe7w9D~yX5qDvW^IV9?|YMecITmQNspHCBkhX^o;b*t~nd~$T~U~HX}Sf@t^fyW|Y zGTrL>+I7ZYSf{o)>&WdzVP%Jdpa;=rTwA-<^xCwI!LZJENT#`ve8x4zou3{ecANC=?i*lM)fdPO*sTcGHh`=I9;7l&F&t&t$!GYJldAm7#mr_hCx(y zOP$rr97t$bO`TF31Y=}QWo@%(63w#+UsJdnk=3mZsjkyZ!071yeQgfWGvaL4`St2L z?W~kcKXk>b(}N+N7|>`d<2DU4-jM|`8vJ>9b~oRdFaf%OH66DU;6R-N95utJbES86 z;E4(2`l#t-u!gp-&H_CroT1h(r{+cwnXQ^fo|az5L$ z;j^Yn@z38x+q-SdA@wHV8)A2_>?TqU2k5go2Ax2&`;)XxB8v$;W(AB?BO^2kNH4KuAgU{NtoVWTg9 zGqF+Of`d=F`=%ZDIXD@`IZuv#($W&l5HsrOd+Wcf;SoQW~v>lkp1=Sdv ziOvGQb9*N;&Y0?aVPwaa9=xf)@&qVRZiLYd52bLcoZ#mESG@P`?CCgHGnsC+olwP9 zZGN&atonG=Djv}~qEGzT#rVkt*Db#K`|r&K&n%B30L|lQ4_t@|te-Ynz&e=NM-6Qe zHnD;~zs%$nBtUxs^Gu=@95MG@_`p~3Js5;P_W)z-)Xrg%+kk^N5w(Lk_pIIc;VPP^ zy_>BbjIHxEw);4e1fFu)YUN=?%Dx~Z#H`0DI4+adyyG0A|y`b(D zGtGo#D1C5!5X?L9^+>cPelQ5VWHOfU^vW&|A~1Kwd=e`5p&@xigA0OqxdeEO+vvq9Xxnj|AWE=VF$gW8)6~2>@`-C=~fq4lfo`8 zCS&IG&45&wRD)Y_WHJ^DK2N63m9=XU23DRXGpk@Uq2U7TqabnxMmTey!VhI^EC7*9 zaid-`HTzI2S2S8BdTy&UF_3FCD;>s*ezIi4Rn?22Y}`)+Ihh{mnss9o=RNU|#RZ(z z3ns32@AJGlwu-35$I*ka6r>atipT3TUIW28&BqQJ)Ggtvb=S)noboUp@$Rc7ucE`d zV{)&cZAb@;($Spb4G9_GQ*Y#a4dgu-Tc-;G7q8QJR7CFnM}r{8fj1I(dN8)m0qkHy zTehW(whPxYGqZyS#_W5+@h^1ig=hig&O@BOkzct}DFo}4&B2n@bB@3H(7jV?BuEV; z+!>~EJ&0TxN-0c(wNYb8GQqVhmpzGIrPOAe;YaH2mkM@y>5z-A%FRpp;pPPd%DwR* zwVgYG^J`n;5$~Erl|)Q{6~n{%E7}flYwU$2#sCDCVoY>L3$K7}fGd%@6iK3icaHth z*sESeXo&GQPrUF4KLtCf?-C|nrw3!}BogZS>N?FsRl#$A`_q5aw38ju96@zWb^RW^ zt=|cqI8o&O7krRDy6f$qZMVt7S~z+L#K z-S_mJi{J10zW+SW9?x>?yj@*gU0vN>U7h4vV^QzA@q>T-^TO$bxhTnJK%U9u_xjbdKYh%l|4FSIES?L)&h^@Xm&V9d9u|JFeR%u4)xG`it2 z^dN!YD5ZVq)Fc{k$i(C7kK2<(j^I(Mdmb8K+OuSK{j`Sri<@Bcs7u-Q#(%Y;&kqj{ zCd|3qxAV{_<^b?{LK0Kwn)`MB;O(uiJ;ACoj-T_Te_i5C4RKCL&V5M1x~7UwV^d{Z z!2ys8=o6sDUZq?7;ltQ;ZdG0~ri=&)(-|rm$|HFpLp7C%)-6;>pM2iDS5N!AF{Rj) z7p?nB)cs4qhl@$;isE#bkY{yLp1gA4(P*_s5viHS@^mb#nRM&jCEV_LXp~Q!AvA-b zdp@6q5|7LZ(3bd|#+Xk00gdu$T>``ds>eZNZ9cRgnOP6wMgnJ`O|O&ZW>-TO2l%!`;$H=J>mX{}n8EHdUUu6rID#oVbW8J`A7 z72wNL$0>QqnE70HCr|k)c9@>9D@L|&P6TEn=>mRWEPq4GA@Y&~R~HfF#oWNA#ucrL zRq~?MxuF?}=q5n%-)~R-@-y=i0LRxus{P%!w-mG zw?f|sR$?RIFm-P7!I)iMzP&ZEF0Gq~Mls(2CTBG83=vy@a#ZBDxQzLnwa{S(G>Yj& z7HSN&?BXXs9x(kt@EKTMX>_FlCiRneJutj~W@@^kq-t6^-M?he)Z(dh{mGEsuwnmi ze1eDOWY!lDvO$!V zo`jL-D=_h00ev+YpT-L5Ode;EppZd%k^J!9s3borjbhr9?KZ!-Z)z9Z`&;{ZYbkQL zq(fzC*NvCj{%Luz30fN4{N%LhkTvuLA88@@$lS$agk${NF-yumOzlgd>)jTQ-|p7S zNJGg^We=wT4KSxd4#RQ^LvndU@{%#|S+n?*pOi*1p9JPp;GxjFFXfrx{nF149g~Xo znEu|SPkeE-^*6x7y4&edTQI%MUde9vT=9s453~RSK|D@vNoUB~(7Sy^UlhzXzuicy zRdkn5m({OxqA^=ZoO`0<>omQYN7toO08pNfv=Eyb(gr&FP0H)Gblm-;u4kTlQWH8< zwk(!p8`L_uvnQYz{Pa7-l$o@Zm6B`CPAj`EnLf5G6}7+Z_#xZXM_RoSNYeLq0vf?I z)4)S1=nK7D_b%Vzrt9B7kn-y7`t!N3RW-Z7s-YE(O$JNXQXqa>ZVyG?7QS~c_)g2Q z2kxHcavblHzD(nmGUmRd)&w+)IUVI9vWIwgm>f|?=`1*<13a}r9s!MFx;F7Kb(T=P zXztAZLk}m+|3aC383#0qX;W?7&%MyeidPfJ1MNQg`Idc;CmDxz*EJEI^$w#$lq?>n z^OANix@*(+%^NH_-!8@2rn-kn%?d8o(eu&`iKqCxOPAK0fb^bmRr1hiRc-=iU-A;&l~X@7pF7HD9@s@+BhZ;t8mvfAN!K+>J_83vU|xSWms2 zz-g^6Jyk9{8I9ZM)we^hWp?6YL{S3sgiSc~z?>AOB8^OD33zNvuk2 zsv4xsiD1(&cIC2bN@gBNYgD|BbAT0<^?i^K&YWbc<#}JS_wA((y2Vx{9}TLa{*<}H zb{o%$(Op7GXp=GfusV)Glj5OCt78k<&hdFG!I|3_^WUJyL)U@hH4m&+H?~dc@yVVR zZBP~331~luxgT)G(j&`7WAC35K}m{&jBFV@J_8!mJq@^_;Byb=^GC=Qk7(IJi$D1O z5K24L@yb(+x8Hw+RTI#sZ>JWg?Vys`>2jRx8UPGD^j_jPDRz%kTtfODd{Iz6e-5&q zIC^G@rubrZ%?yH}4r32$KS~_M(-~T|mpu`1O>0T@)=i~HZ6U>)RL&vwOA*WV5g(OL zT%?CD(m3l^+n8>-l%e)rR2Gj|&fbAS^Ofbqd`;f&cB2$N#xQ`<`n?P^x{KPnlL@cH+w@*R6$gW69;A7qO%-MT54%20z^{T3l5* zZB`lOg+TO&`eupu(rlkv0HFrntB@(ZmbMh7H7qTwE}1riig4|wtAWzrGv+yT zZNF^;8ihsIHV?BYaQ#%%H%hx{CLp)AQ7B7gEd7S5uvjLctS0^q?Y8!?59X0%vAkkM ziRpB=SP~GqV!1QY`he`6g~jUOHP)>Hs~0c`eC-g@&Sx2r>(ATv{wuCKFcl4*eEv^E zmmETx`~*wT*^ATvIpy9Pms0ZMSLQS*N}W&VO#^$O*A8WtPPyW-cXp+18%A;6oIM$u z%xDV5XObj(+EG$>X;ax80D<$k=<{y|ZN8ZtQh9+T&`w}hGQQJDAyazA9}bPQ*Nzq% zo2dtnXvh&uyWa2`jmj&w{l`mIX?kVrSzIK}t$~~58`R;IReozl_1MojC8CKqNFeOHk zm+UIM%{;j(t4}H~=H;Xm+pV9~tI1OxQjxqUAFf~fc7W4~(i$*NN%~u>XZ$@Rs;Qm7f&{UoBrIg@OyYv+3SVL4inoae{Q&x68It0yCQF#K^++05h z9Ds}oq!rL)m$FSzN=@~&xFDM#507|%ha&WNqla!LB+Y7%4t6a)Y8&!5R694HRWmLx zB*>b-A;|w>ZYtjRLNXfK5;tW7q>=r!t~TWF9qx@T{7=rzYXo$*V20>*np8QoUUkhJ zxfodr=`Sx?fugR3HDp6xU}mk@uxwS^T0SqXuB2POry*n_&&S};j)QN*=FTL+XASAP zZc){@Q@vXrIE@CRz+DSyfNA~YOVEyuYv>}zIKzQ6E0W(*Z<$0ba8+u0B)l!`JQtVL zRL|h#&*b4!HS&BdMVh_@Vam|E3(YjVje}3MG@IY#Nz}N|&UARX z^VDP4j5+g^3A8Xi`}}Lxw=K64`V*Dini^ibUK{y_N5otHsZ`W&!_JLA|Hsj(FlC+Q zAK~o(JZIVPhgNRGqIVDoE zf^CN6&XjqfIH|4d1-44*Rg5Ykn>p-IZe-{4vJq)0Ss!s{ZtUXap;62`sl_%m86K6< zlJ^_~+kO+Ii-%h3k9Om$pxdt%41(2g4nzuuX#hlnYji&PxD5}3J z8J4AE+D>6UO*XYoe>mv6f67#`aGd6|Oz57C7vJnPZ*=F~)w#Dzb>`V53q19qARqC{ zGof1^&}g;nh)>5xj<2bt8y@}68*EKwLgfx<6n8CE%iPN<=$rta!O@W$Sv!EZoT9wV z&5?&*@z(+WFZz^twmVLE#D{6RrQ(IuiKGRmU^3)qaFJxo(XXE_{AFH`gV5XT2+^D% zYi(~MQBZG7XK9Aa#-i(nc38ISeY?75L9By06~*`m1!Nr}y8d@F{ozpvX)NdJxfu9> zMyt0MFy8@vp-q>_ITQJMqTN#wCkGl$m<<`zv>Y*+FegGjeovEJf${09 zjyX;=Xiw;S^WMF`eCA+E<4x#mHjoxdS@qMfog+T;&;U~vf0LVrAXbuwL!O1iYcBnH=_Si? zbF-k~GvUUF2V(5_1P>$*WjGz?WV$;^1r{jr_!c>Es>CwLy6`&?LO@d&2UTU&m@?6M=`C{5j&$1K5vk)RFFDpE zWv4DaW7Y@*&FQ2?>8_YiH7~8CWL6cWl)!2h_7Gg{WW`07Uc#tr9>ZwUODoCO`N1s}G!-icW2H?87TQIh9JCK~0pUb^#4a z-5S!er$ggVB)9d!(xn|8?rGTddzGzj9va1LKrN6xop6IC17@YI>%ho)XcXC=pO06O z9Vc@3IX+LYlbs(XYUgk41oVQR#!>)XXJt3Wip$A(lo<_T_LD{PD6(9J6H1z8;##-_ z!U2tzG7hD*mr^QekD2%IWjV?WO_vK-Ro0kk zlqx#T9zB3_M^pBX0deAM%G6dxYos={Dz2BPD2r?|Atwa!8`Ocwm_OL!<)J~PPW%7A zQt^#ZxH`+$>OeG8tP05{pc`tqOa5+?02$J?^wt=|8o3OadA@-v)Teohv8kbgGE{1! zcNw!4QIlCeBA*s>JTL>Rq?V3{$RDkIt*l(VeEfi3d7cCOIB%VkgYwMAntgO?>3Cu0DE4pj z&@28D;6G2F=8y?(L5M$`L(a^DN28c;5oY!nkmSU-o@9g^=daMGIVA7VYblR_^VOjD zTa%3oo%?rZe}cIEfCiYA(0+tEmseU3RV(2O)D>S-7dGF8yC4D=*|RrF?V1NT;QFg*S&IiF?TYFvr^I(WA|>Q zo#!NK;(Imf?jLme&>JV5hQo2&CMWy>jY{(sNXMp&_E+?p|MQZ=zTD=xRP^kSWixL3 zV>V$1L072@GCys*!(OSVf5p=Jj}+1cXTiZJUXr9nd3T@J8gx73(3DBD?qC@>zEyqI z!Kwc&q&VU|U+=Wd{39l%s2S)u00n;$W0cg86T#Sg$0SI%`m1r(5?8D>`WQ8uLlCqL z@1p5=&5y#!HnwQjV&ZzFeTJ!_7_mGwq7&)0V5Siz-lee@F&M7R#lKcZJ7dmf=_fr6 z?>vdwlf{sy%W2WGA*v0a_oL`Ved0M@fpw{@@p6NSZ@kzL5DOB%U<%mIS3~9T(EOzu zfT=8;0Tu#RMeUosz*Vv1(^*8H9oFcj$`EQDsI?e;W}1R6KN5($11&JH4pV+Eb2_yK zDMPZ$cr=Q6DlpFgeW7>piymHLD}y@$<3__u11~p7;EB!zrFLJF5i#e~X}zWU*44(| z8i#UuEx5!LdcCaWUjRqS|tQxq1< zBpOlnv$avS?R!y)GS-fPF!9hKN?$Uj3G1I@O$4WQFXW7%?4Nqv<{SF|qzcKtMk5~U z*=onx4&V%f0SS3AeZ~`IAyazg%=PgcoB_tyK<0~TDlg^?7OSftm*kC-A@lOFqM$Xc z>BkL#W*>SG^X}j&C(NL5vun9L;oA;-?W&uOY z0}>B+^<9HrSB+@1@Nm1lZd-FQWENi+Hwy1~#EnSjh+& zSkERy&+u@$j(BVRPrG89@XSMl?P&{Xf~~k45B&2{;&EsOQ0Y8Q|HMtyKB%VR0&x5- zaY@SM`m7p2*?q}jc-0OR#_@Vmtm{17g!Lp_Z4;+lSE1OKx!9!DTkqTZI;@^YqgAn? z!E`ogDz28s1%!EfdTjJvtVh_>o8_Jkc@3j|bR0U=w&+|merf08KH8lUg z!8!A%*pp4KlCaY|QH^Sw+hozxD0w{JFqY>-l`W>7rU8%7u6?=XX-!-<)N(k+0gYnr z3TL|q0h7#{5O?+w8-&7?hJY((9va0Q4oo{hhkk@^f3Bp_G)F|!Z>t~T=z zDKS-8wswqb_m>S;a{zb^4GQ%c!4;PWfm`eWuKw<#si%$^oQhuEuha4MXH>%%w9*THkE60Ev>zie2Ke=<|OiKQ;%Qt`C)q1uwEjHyxD^^t}B^{F_0vS8twt~lb z70g4gO}7b6cOUUBO$YlGd#L0UIfpH12DO+%qnO(hyXHVvF=q~CaUyzhbB95PHNX6I8`KyuBBIeUslYmAs-TW-BI}~5%JJId?-{0Fj zNqJnP@9*W=7TZUxY&kBheoE;T-_iT{Ie6fK?AoB0K4ZUY)fg3NxL3u0H@N-s@U;bcI(i(OuY z^7|iRnJ6>#uTyixu!p9Iz*}P!a18tL>;XNP(G)*sDUC^4!;!}0u2}iWr3d$#N9wZy zo|f+w1T>0iQzrcadf$YwK}gc(>B_tI%WDNNo4Sg z&&D2NM{ZjcOfb!FT6K`A&Z{e=HeY!-1cz3s%&-biaAFVehHriHAy!k2Dwzf5q11u^3a37 zz2Eh7B~RxhS9#A~JAU$z+rD&y!U;C>0=K!V)WeM@U3TEwC>4ELziiSSt5K@2B$BsmHxrPhWD)PZ0j|DA!Nr8_)U^9vyH&^br+ z=^pE6J~()A%G5B4Q#Fk`oASfHUtFBpQClVbAHY1FtZky_;%hjhZ;O^Y_%oLiFw>)=xznRP{Bui>h!mf(}NPCWOb*q4*V zGCeI~O6SmGT~z`at;%D-{DZ~glwAD-$-9`NBLp;x*#{kAJ(@inF?3A}8|Q&|Siw9R z#asnGze{4WPe6g>MO&Tn+s4Wbgc(?&fJQOxsl5}Dn7m*^k+^4HaQDfB2ezON8(83g zMlowq_Z~^ZqAJf0BIVi*4NE|ym|y0UC$}l7lDuS0Pqd?NFXbntQOvLMV3O|fhjcKj zE=-R`F~81nBM~P#H4;IJ)goK zMSKSO8PF)E8(00l$?~U&*mkG1ula(;)f|`sjbgT@<}r)7jhxL<4{Jh>c4ML)MnPm;k zKM6H`94%+nHY=u-WM!tjtYj{h-WI~z1dta@mYY&vP9NKdWYMyi79opgr7KD)_|YIyTb@s| znR55(mQ*2u$Cf1GfVuP1VvxK@`~XsP(gKe)_7iOskx;#rARB zXsgSMaFcfXuD)1=EFPBDB!br+8#HU(qS88Fm20Q_u%Fo&rq{`arw@cs+D{`X7vnjJOL$HMb=%EEwC*qJyS1%tm5b>VUz(JI0`3UjIf=}IcZ?#tYOV)S_~DF_|B1G$ z-IJG0$hrI!FB`s>&!j6{<#c(|mI`+! zNrx_ZLAThJicfc=rlr&E7LVE>&nMZuxSlaLU3S8jIJp<203NyzsxL5M*U~7F9!$zX z$o?Gq$$uh`3QS^wGGw2xP4t+pO=?x_IC;5)Y<;x40`|BHRaW5tDybu6LI4Y zp$2%w+beXIo{{-1{Z+(A`cCUd!d>a#;U*vDvrig$02||c#o6;v0^W`gd zkV_g%cZR9a>{plJt449SYF5Y8rA&D-i@5K)!*DE1S}KwkgJa-?k{mld>q79~?V7SUIBaGHRxn%Kb!hL1k7YtVL`I_PYs$ricEZp!|DyD3X5 zB75?{mCoj$R+IS=;gt-2N@=nnS{}STYnWjQXfFmzrm|e5UFD_2!0@9l-Q3bWm zuF~Jt&J)W_Wfip3nGjufmX{oJBG!$+B)pv3Ma z+TBUn3DskYGm*ctpc)tnAXPNPIE74$!7hBN3~5prGSg~otQ<^TID{nn5UKt2^Y zKxva6BbCax?Wj{VESWQhlBB^G;0(7+w%?+|tFc@>GuA@jIW*OC(=*a;jdS|V9_-;dS z@rxfmX23yh;4xbIw-}$+P_pDLSw=MPh&J&91CEu0eI3Su%!X4EjQr90D3(E+DFODXoo}^FZ%+b>&kW+DxcO0Sz#%utr0| zW0HKT;_*wp?BT%lW;wWe1@wxK#xwgczH2+CC*MTeet4Ct*ro&H=b=}8G`887@tv7a zesqf|3%@Pn=b;DwKLFhqkCW*^GI#XS^SynaU`z>UfH{mxwu8JG&OAFAiVDT~p-ARt zRZ=$s8epp2Ho0AWT=H7MbI#e-_&@#mzSl|BSA-W5 zn_j{szY}potPxv@i(}&3MqDQfXizDO`8dhFwqbE`IkM&ZINF}F|3nkK%=mC z(P(MnlgW+^w$yZPn?G!Abv{uwf1VT2D}D)d{`_P=)wbggX~oO0rWwmR>G%)mfqxdD zXOSJKZI3x7wQN*|VLt-LKcL+{5O!-6j z7C(Zfz&l%`0|ZO{k0SFklekq&UBDMwb2d!F7UzXH7iQ{XNZ5*p8MoM=&CKFlPpW(Y zw^AM&h|prvxkcfq<&@Sdk4r`8KRj~9loeM}ZR6u>G+C%;^9n=(jbg6C zFtaL&$$3|Iq+|dmHR;ghSjq;wEI#kn2h2tpD%+wEK%?l*%**?*WZs_5|k4Bqlf8ew5~I^KRe0{1-6Cv6{Ma0~*DI zfi;al@5`7qfZUG>)6W<;$n2X-#sQ6DB6XBm%U-+VL9Qf^uMfkOZEt|4z^)AP@}O!m zr-&PpGJTlDSq|x#v8Pkt8fkkp@p2R!+$Exkm-t@Zu~cjWh|pJSH74rNewpNK@XW^! zD4I*>b zgkRe@U1iT>mQo@6K+Eyap-f-2`W5Icc<4B(9>>7jGMB<+z9NpGU5ZDBLul$X&7+nS15bPhJpms(OU61!Opt@dxB7n|$W z+Qs;~rVnToa||$JL+OIK^t6Di<7G@UmPj5N#jGSusv!nZ8?Fw0rwPCPuTqvVzDg1xxzd1w^VUa0j?Qkg7mx&7VZx#PRi^<}ds zzIOQU!-!AmSgu~angsNU4}ZuUl$0&sQBBEm`<*fKjCS=qkq;7FiyY7Z^K@v%8F-w@ zYU0euje~dn@$MZcefopt)vt^{emKeY2hjKiAfOSY(KIN3bPep$h{T9(Z%8z0AY{L5`J(MwJDdFZ-H2qo)|cJVO- z8pX7peqereC0kx{V9HNw_7!sh>evhkYU1ozq$7@BRyW$%Z1(NIi(fB)#`?friQ{x< zE1RmS@?!s6zaGK2TIRF#t=W^5CaWxsZ|A6<=DB1O&?x50oS5og%1aK}$WQTrzLm8Z z(82Uuz}$=ZTulGK$I#hulU`r_*J0c3`8s^o#zaVMn1@FB{4gh`)=OR#)3t@nqSYe{ zRY!IW;@buOwEIQ2DkI6Fgd;aY`hhxOy2pS z?-Q`!NVm0|{M385mkr_@6D|KXl;8RgzJCzVEB?uZZ>Ne#b2+Un;oIC0Y0d@K)FEs108tO<2gFXI!Op@fq>r7hF38-jm$&bUC zYi(tn!Qhttp-9*2GbX5suEoep*qAw;U#qyTWB8Z4j9##aZh0bG_U^yRhrf%CWP*AH zG+HmGq`uOz5o(!=M$By2=htD#Y8=}~r^o?~V!96IcStz_7tQ@({ln`P9YdHw69qJi z`7kkDxAsHsu>;a8pehkc=8Pn;&Q`e7My7F1N2(a+1 zkyq|(*K*!U>PbMOm`Hfa#?hCjPVu*W{Xb8v9D+_9WN{5>fY}}W_cIuvrs+Aw$aq;+ zg$2`OGxVpVBv%hvyW5z32s&`5xT!oEfwpmrrU&8Yz@(h3-0?BzzL3S5EZ3t^Oso)v zHivJ)q00d#Z0L7Qt5*kEJE_$(CCNS?&?|m1k%m00{pPtC7VR%e@*L18=1~-A8}iI? z08LXm!OWPcNj?J_V3HM$w>x*&h*#gq;!TcXa=rQ&PntAtG>^YoTO3Z(?A%b6m23Kb zJh<%=9JC6QC!opZVmIXid0?ld=g_K5)!$&k8Mj3Z_I~relF4*6Rs2F~h9TE?0q+0D zg`2+VK(i5f3B4ORf)L*bILbO|PFh9EXL)5wDbj$_h1SL8loJ#gyZ;WyPHaQh{Pc9= zPwu6gz~UELI7&92{G>E>iOtVooRHfjds$o^ZFh3{CEb4L_Y8R<@k_=WnPftKQkrZg zTv+T0CT;h@C^S`7Cr9bd^euku+B=Wve^e@x7aixF&*Ym*AyfKnPF$d>DJv=6vKX9A z{rbXPn;%X?QzS3Sh;t5oXHUqKKAVv}R88QL6~iY>PfsI0YRgPH1sV47_zV73K`VCh zqD;Z#3q3a>Q~GSCdWYdg9KgyK@NiHAVz*qJqIwKG=5M5JlsHxhA=`YKpNB6mXxgk` zv*wmF`D+R>la1qS#yW?LrK>7Rh@tElLph1zj%0+vcL7aSQKksG?XTYIJ)|5v(1&c- z=p_3AO%~>+t~uDRm`=@ZV^vXwE)W(T8ib=S8Hs|;Lz7+H&Otz47)3Utj55jBl$Ym{ zUvs+Myx!oCpB#_L@72k=%1=s@U3kAO7oM)-0|qF27AlTM9MzMRFEcLNRXo(BI6^q7 zNqU^}y&}=a$tVerIB3LM`KRH|hSpA@y3)CwvWhB7AB^KWG&b0qDe#E*UA=u>^AO9% zMffm(-sy^L*$myXY!K3igBRS_IOIILje*BWwWBa zR$ej^ZkTM3I)g|5mbw&Ar?HeG9sS*EYDoL%Qbv@c8P;%Ar4pGtcbIL~6E zCG94T+Y$@D&lXV_=p6JR+I+$Zuc*x4ujF=(1LRD&p&ihe4fIc>ZccCkNj8Ihx7>!r zAhMc4ZaJdpc$P^g%P9!${~*|w#WkR&I$cg#x=YH_6tG@tY#`c)RJGc6@xPUE=CxTT z(3vx;{{i%yeZ<{O_bi_%3QwhMK}5s;hKSe<)3A$M70T1I%3POoEy|amHlXw}N4`Fx z<#ePK3Jx#GPfC;BT)X`ZUaAn(Zy9k+IDKHxpH2kG3sz~Y!cU_i%kG6z=$(bY#!8+K z!P&87GR(zzTms`!XwHJ*@zBLtz4$ywZCQCJzKHW(bV`{8^eGkVJOi-_8crX!AES^( zMZ(FBSL|b5X>gXFjYXVd=FC9`p_HBjcJe4*(91(FHBZ|FhLX~XStz&4?6}kNdjD?Th4gVC zyX3Ae0Sz$iK|ky#n9J*|yD*cf>u7`5B^47lVAZyCCO5wHwt}R=8~K6Pl`(^vJ zXcQCYcuhAbkB`a9LvFF-GtldR2ADV*YY-4>7PHbiZ(QbT(e<64X*72;&5VQ|0f4my zn4(c<%<};X*n?ora6Fn3=ft!#2pXazt6n)qA{`TC&nlc;iUz6UZyF@UDN5rMvJ-c| z9D2>P97AQ9zXr@-3&=v#4aOCZSpCWIm0fm>*Pu@$1i7{id1tg?MRN^W7AnS>MH1B* z$Qh5=`ym@6i*qJEs+aWSRtw1{yUk`9va8oim?0Mtz%5T_QzXIcmPaDW)pH8V!EvUo zF7f~7d6eX@qDilgrCC5B5>zK5ar#UD-~OqCprJY( zH;5$KS8)AY4-?(gZcZJ{4m-|P`>miX$akT{ggFgr6fORN%C7B5S}acMklk=rgx55v zt=+7c!PkqmFR+85?Fk3NPISn^s6H`9C^Ytr0Ula*oF~<*dXD}n+GlQH={P0g>e2yY z+4u>oD}qn~n$x%gB&$RHj^X$?q^(Xh+N~A0o9i%Y#UwYJwp)_<<$}4gNZQQ}c7D4t zmo%ctOIQ(YS{W&187Wy2vN7VEsRCY!c4vl`KAP>WY;X(44MXEbc5f z|AsvL!@w$Z=lt~H9_XxiK%kqqy}{Qqzx!9Ir5~)PqBG)u3GvnD#UbT zVYrU2#*91c_vFOK>&lLC#T2R3l@3C!vR5pwE;0XMRd>azyfk82bkWT<{DmB5f$)`u zm`kSyqSdigpI;EtzDDwi+wWk42iJ+8`ZW?iS$^>12IdCULqVQc!W2_DK@?*!dY-Gl^4T)eh-I z1`8|1izCsdlx+zWWF=x2Ze>Bbc48!&IWaaJ&7y|-1+{+&Hlmd*Czq*j;cP3b8SOPn zPYAoI{E{^g+G(ymw{|l0z#j;uViAYT>}bP2{z>elH`6fASD}U7l!lBhy~b=jee1TV zx)wf36InFfY-booa^Q*Z-Yk(K`~x0(qf~VEbL~bhK4*J7)3%xp589l;s_Xn^796RW zY}SdU!*Ma^GWjyRGPi>RzpF8#+O{=C=1wNb;h4z`5zUNBrCXpRBuvP% zGXZDp?*`-A#<7ACh9dH78qxpX{*Ws*G$HZTJk<#T74jac~3 zT~#wJ-45z#g%3B`oXOJ6Ll1n*b59~QxV0gR=S25F4&8Wv+^MzlAa|yi4_H)=MAeih zHym|q#c!`k3&by0P5X8Z@dq?2s(*W;aw?#rmASKu3ZA_=heef#9{7I(+FFmlxOs7h zufZc8rRXRMJBNKqRISM*kf!FL0VaK{$s4jZ@=D61%Bud=bgOsqz*59&k=_b(4bZp z6Zc!20%(%IRVq*sUvooN*X))YXA^vGERu~dC(J?#+T|6>_BqA_Hggm=AZ~$hw(dKs zEoCUf&R#`jMH%tuT+&3@*jl=etwJ6eK~A55UNwT{=*DW9KgK$?9o0E@GV{X)l#!Pb95A0wNUW4JXv&MhIk_Azjpx3hSeq(v6O~ApJaeoP742A6*}J4; z@odr+&Fqn~bJ1(C_G0%{kLng2+8y?ARt196q7*ZC;6qOV+6YGhbjr*Gkr5|eX-YLhV}F4 zQ9o#p8HHwvk6zjS2xAq=5n#PsQTvx`3R#Yecj9+Qnd=BJOROv=it?AWVB*s1?0?Y| z9jSUa=iw!CwIMKB5bAc^9yNY`=ShJMD+(L8Ue=Sf zoSY9K`~=4Ri&W(zt!UpxFWvRXtzDgcNlr}2iR1E{q|7y3f4B9ky+O|GP!P+U|NTr# zaS)45rNlu*RRbuC94(4Y!!gR5MP@}hfy|h@ zlD3Z}LCdY88xqy#v_>>tLnzG=EVbldKocu&wN|dN9z;#9XU<$_P*W#U+e~Qd0vhQ6 z>2{=wrXFe%b}*etV9Po1cD47)i(=xM1)2cT5dsPVPnS}9-gNbR!l#K?oL@sVLtb)Z z9Fg@Cw~!KK9O07Jh#lbc%eE1BLA7v3f{NW5eH@Q?3Pn?L9&vuRc7x=}UZ>fP&21if zt>R$d_hq~2`h?m=VGN$N!`~V7dF+ylo=T1oG~|kMDGQ?@6$xFs8V@-87 zLz+yd73N=C;BC3e63u31FBEmxf!|!h_&ElBv65mxoMtnZRCgI> zVym5=0O@yTMAx^pQql}2aG4cRq)8T8+hdwLo|XY&APfdDYqaf>^8@z&x9o5nOs1VezDSs@NbNo^Btwm=s}0_yxJXP35QxY^`8l-JCz$f||vzr0ZUze}PqxPX+r zsp*x5j;9xw%u6I*7JZ2g?XRwZLvDW)5I8dVsDZ%w!IK+LB(38ngmKnFItTodxM0g^kDEuwiOhRu?B>FfZ};49Uq#VDnO z`+##;WDP3fR7kX7Me1NR>EBe!rY6IqvWl{Kb0{p@&y=nQ+;#K4j!EEp7tvDJW`yTH zNU}cHoT8&T0%>)YqdKdp-;vN9y{wVC>X;{!DiY8r=3~H&FYk5>Te1Td^{-f3|B*sE zDi|IX(5dz%FP<{?@qg@+qPDV9e3b5BrGvUtE9Vv44jIpDeY3)Kr14ADYf^GTr~Tbu zQ?R=8c_U)^UT#$<235zov#xCZ{7+aq4Wf1e8WqFa;JFdf=g?6%60h@Il#jhGKJJz) z?`==G|AF3y)uDbY=GdH{B+-7Xy`_#_DDv4v!<0K52{2WF^c!A&rUvM^VISleSHn2UrSDmO2j=Vc&=M5-?@<|b~;9K}lc+L#+i!(@RZM`N~I zKdV=hr#esvj~8|G&?sgHDD8u2S>F)p052h%y6EJZHlR_=58!$&$~E@7mGYT~ zMlo@Tg&D$_PF5mm4JXn?{5&*@iRL$VB{5m-WQ673>5Qng!whH?b0Qq-wswAl3pa{O zC^U_H z;LaUeo>^KS`x(JP6GPGbMtZC)G$mVgy^AB3VmVV$yfm^uSimFZjuQMjlXo@uk~}nu z`7v-}|H|>8qZKbj?@CpP9iTikz_bq5alpJew?>ouz_MRMD^pvsjdYmulhP>WaA2;Y zwxpit#zqcyq3d-%y?EaIy$&lSK7$J@MKPoA_CDHtu+uRqvxvo$Y-wUmCRE{-irjFt zBE0=Bm*2nr;(hJfo;u}WZMDBkl!7`EuC@V<*7ge2_FS&5V^d}w(K%6*L0k2ughe93 zJT!_4@5J23t>WlKx1$%|>@{z6=iLc2h~5uq6!TSJ79{N<_jg)gp4VvJ>ra)ziHcj= zl^f6~CU!f_Wz47RT&$5aDLmnw_bxhRI_2q=TY&S>D5l-RiF=N-OWGN^*=bSH#62eu zjbe_bX}$Cu=ML$Vl5Zm3gZPZE=gdQ+m{=+>H*nn@z4GbWD45#?G>VDoj#-w(45k_x z^BeTdpEd+EiunwtBdzQdik36n&YXr2|60fCv^#KHr~RF5 zon8erin$q>1L5I@0|5;#{NbCV12{sK!wdFk6thKB!W^8$6f5E-+zEDD0~*C#0L;gd zeM-GY7wa7G8PF(ZNpr%)8MUmstF9i|?4H%Hq=&dZ<>VRAC?+N@=8mL2Q48Mj%2SKC z-yi+-Y}PA>8PF(ZcVM2!?e26*_Cw=Mt^N)(4~=3@2Idn*+Am+cE~53QF%1Lw83|u5S)x~6?n%Z(#0ToQCsRpK{+Ss z4ot>VFt_5XtjUWOix(N!4H;ya_iQb>#V8l24CDF>#Bp`FHMap}r*F z3u^bhyVJTh&8o0;h$Viy5h7FjB07254R=6xR21rODLX!fCP-)nS)*QN8&DCRC$ zx!RbNvAC)cr(wZ4;3cfqPR0R^Vh$rrTHmlMqd3q(>AsNx(0$6{UhS*eZ(zY6i2@ph zUkH}(XH}~cldSr(ILyE`Pfjt5{t5XKz*__FlWXsg7iGuM`-6-+ddE zcR-_3a^$9NZE4mM_UXr!objxyw@xOef+pw+^^u!7f!waFQlMDJQhgJT`O zGij)R%S<6hYe*fnYOP86%?4gEcE(DTNnWr3ZkO2)L5Gd=5agj(d~_~zFL%an_!bXc zzx3jPf*=^K4QZt~DWQABB%VEsEqNL$Z|+NW5gkqBMRe>)Sb+M|bM$);+XN<^h0}=g zhA;c!^@FB%w=)vkmIprnOvM<&S z094GVfq8ffHV>}Vkle17+P63DikC8F9!X+~>~yB49eWvgI3S51&@0c5KI}znC{pm} zL*74o0!@5HNN)h&?w$LVfOWv5SA0hw;#0e>E52k^`zlPVFJ(nZ_GFJnG5-J$$XSuq z^Lg8TQ&B{h3S9cc7f6>qpV}%*p8^`iba7WcpDZ_8)@fS%+4}waV1p&-A^{CB|4AjA zQ!xgv0uTO)ZI%_~+vTYd1^Sz9xV_#fKwiQl& zn+iHGsPT%34C7R3L6?R4faUbRDDKr@vx&r(hhEFNggB>Tn7D#It1QPSm*p8oE$b?j z^#qrdhhEECMrCo(gg?{+zoD-N0nM$>lLUPYIUIGNd*oVOuy3B_>N@7tdKJb=hP1jb!v%}E z&qJ^HhXQ{b#P3^$ptc!PhcbB*QOpluxNQ)wkI9{q=E@QcO1YlV%l!@*&?sgXtZA*H z&Ov-uppW1YZQp9l*6B}Mq~Zx?FH+Dr$wM9*#Z1$_rg;H8_ydgWy+!rUefY!1(p->2@@kgti3Nv`9~0Q`#?OUS@%~QJe)rBdF}7 ziP~LCdPLy|Rq5Gt=&~Pc)hLB63DbScjYz#+F5NK~L4G+&4LRvV-Bh5Qbyk)U(`I)qhCt6>o2Hua1na7zU{$hPPIj4C5#2dPhlZiGkN+;mYd z+v672_p|grg8ms+r9AX1B1iA%;TA>k0%1~YJ>A?~wMOJA2G)%hYA|)Qh%a$>Bx(di zmFqfEtTuoHaBXFOL1p)1;W$r-8Pf$JIP)u6AUQ@@AvSw+{hSJr8rwfN!qyRH;#t(N zhbjtJl;^RQ`AuMcI5Y1`WilH2Wu(oR`4@(t4ZF=rl) zVopMf4T9GAn7S>c$AH4M!hl9GuK;GSEyY!*uIcho1lv*pjbbhXW^CJB*A&HfJ%HRA zGJ0|R{7Kx-dFU1YX5f!u&2??T>P6e5Tv=p|i8VJ5jbeW1@$7nJph!7}lz>Jt?*ZnM zytL#n<#_W^q<^hdjz639IUv7Ta zp+!^Ue5JUDHZWW|Zb?A7R11$^*is3cTk7X8I%O(s44!+c3k9BBy z!eP4T@|=#Z9IMh69EYX|rW(q#(5u)Sy>C5~XBRm1a#9Wfjbb{D^)b2W5J7L31+l^u ztl0)Mia8PWva4u5COg998L`*C3t>K*R`*F(B@C%! z#^5c2m{qx%&!xaFSB9^^+$a>_&YchWXM(+zfJOz<2}d%68K%@R_Lp&uN)SyP&?x3{ zz&tqFd39FIk{m&p4kE2_MpSFC`ZF>=B;A;jnirEZ{1M#fY z?KH57rA*b!tBZyocoBAZ>zG~Bh5Zn?*B=4KfNavOIO3&2@VY= zZb3&)i@eYkFDxN9;HVnPi+P+$b|z?l0hD|TCZQ7@Pcn&fFoWSoW#Sf0{9%sUno`0r zB*k@+VYjtzGdTX#uPX>+gIbDs#uf)_XUXpiJ4Coc#t-Jb8w=|R6 zQV{+x0)Bk_&B-UASNsN8A$ldLhcYZj3)h^|@4dUAGeLw^K%2W*!>Fd=i+4u6?cm{m{r=gGBPLVb8`#vhlX@+3iv>cvBvjdRNCas_8k&1p zA6=sa^dg*e7vNvccF57IFy$pjn2`LWOB;%b!+jFcLwZ-wp1jqQ=$a!UBLjUN8pYg~ zFe&T@VKG%TQ)!QbY_a=J=SuRDk&_R3XcXH;Nfdhb>u}?56njL6ewX=60P8SS_M177 zhgreR;8+MCl?8WUgwKNPRC*^n8z!|&`@D3#uvqolpYH}9*Ssg#vH%_T~0&7 zV&kLtIY_wLboEh?PmNKQi|@D6PB5+BJ72>0J#^Bj@ApbY9JY%I*e0&Bt3p7dRsH~&n?diV3VHCnXDgfb z+b0$6HoYxfcYdj>^0r`oTffS5oz%3-s(Hm#^Q&lC46#tIE&+~~3mo$<8+%7Td+3ux z54vu7rO$sq`zor-8?4=VXcV&>c8-b`{Y*BL}VyuI=M7f-&4FpJn&Iqm}*#e`Crx09IC-=cG`pZW95Q!lYC0gIHr zgA}2*9nM{rRm@Erx+)=;HU;0KJZsjDZA9gDbuf6bx%cS|L3XeK6&{_l1p%~BA`*s30Mu@jrpuV-@-#3nCSKLuID_t83$FGutCa0 zqnIOr*^G^xqnD_pX!GV@*W7#HfrPn`dr}7F;-Pm~X+XPp4sM5;#Y3*z%y-UQ@!a*J zZ3m_ni_gsV4-=kLO|u=Uzuf@hzD3?eHRXt>^e4X7(vh4FLmHfvisU6@-hpKNJ|{mZ z4QlO*@AEf`CzF(2ZsGgq_IdcqhY&vC;`KqHvz_JeF!(Su}_q^AWm$v8l=N)B0!>Cp(LO)e|<&O1!;LcBYJ z1I&OXOaA6s)?|mthurAyWX<~}va(rjf2XFS_RT}DJg|6mejdYc@ZE57BYhF1xtj^T*S?dNt zl2>)4W&h%N3#GFQ*$!wFvlQM(OU87@lZz%FQg~3zJT!`leG&6v zl2279S-?@_>6##*0p=q#KJfKKJ0-(Tg}swnD zixSFb9vbCy5NWyTwFN$D{ZIxDSNB9+c(bV?>V6Y7pJ~MIQcJw!p)oz4S*QYyZ$f2z zO=-aApMbDw%3@Q~1%==dv757dD*A7~R%=Uf0-}E{k5%cYOD6 zC#TFG1ejH<20`8b-51MRKDfUfI9^4uhtq@BmP^Z~)`~c8GeVqQ^QcXfpG+ zeW|TJO172GQe{oiEM$o}HBu4|SlccYYe>2QDMTV~hf9Rd3v8WJQJZH!Yq9H3eEFS~ z^E+TRkn?~;CLULR+@7iEfYc}7zCDPJO)g_aO#ezEt8F)}vT|0sY-ULuNfl=obLGjg zpJ-!?b}c5ZACof3PMxJ4JV$Jo(jg7BT);1PA+_{gWr_M_>>KldnY zux{!ZB)L)Y!*8U=_{xNc|{#QKIFIZ?S`k!aOQFVbRQ4hoQ@N$ z?wT^Ipg%qe%~Ps=>`4{byYHXhpWC!1WqNYEI=*&OG&`CLPLc2ebWpy;s*{ zx1d?COn-CqYxzlOAU}(_h%o7@w~FbNr02&#qIkH)r$PxMH7%f|zOmq+D2`EH!Y(Vn zD2cY)9Mt7olG;v%=?SD(_7Oe5IQsU$AqS_T!(REQ)9gbhQoEgKMVLlLd;5q}!|zYm zM($Sf`^s&`r=)4WBNBs0Y(B}Cm(0)7wDnQjt>>7sIVE&MG|dF*0yj!|R8|pH&!D=w zl_yy*(ZzKYm`(e!KN_vjGn^AP^t-0jt7I4U?A?8bPexS`?2zC!I)iMzI{Y0zO?|8mJZ3jBg3bL*_y4HWEpYDml8;FF-_%fI$DzSUMLBV z=-12J45?awCT7`+dlN(0Al_N7Dyf% zU|Re3Gwj>Eq>$OyY?z950oe?L;VYm~%-?~znktdiaN_RAc?gk5H^2;N6mtlP$(&2T zS(swSq8kVA`s3X@V9*Fg?|?=zcY>eYk_RFuD|VN6N<}-JaNXLa&mKXT!MGgIDCT@% zp2vJT%pTo~M-`qkl+Nk2y5gL18*@4$0gYlJLyS2;S$D~T5r<|Yf+Ub`&Zhnsx3^TrGEif+IEB zq@oj_d#rTrlT)lmxS4E@nGV?&7SE`zoZ%7f2o52A%vLuK4KUjQa}6F%s6M2E2cXt?xH}{F|9Q}PpD&tAn44H#^3W(|Cy1wW z5>x%*i|_lPPv32U*%&nb!GwTDTL%`bwqX?=qpv|1bZW)5))_Ts>491WG=ROF5YtMVg03F*^bCvm~b2sA%9Z=f2gg1P3Go%zy@%*P_+0!{d5- z5TEygPds#|O-)zEe96s}hX$C>5tW@-)i#}ODrXl@FRv`ACTOQIviqiIHFIcHKqJuW z7K6o^ECM(7NG9~?phqj(Z1YV&62Y3Je-O|Jrg;RI&!c+21(sQ@=O46UOlPG78o^w5 z3^3y!Weq1D@l=AwUC!=Lk@~*K%+EZ6I9aj3N{6*2GE~Snz-U=9_^!KzVnpH`njexJ z|9iyr=(S-TXjZp;kHs6`>SArHZM0d$U~LJG=-h2qf8^s_+oa+t3NDrp`c6Qjz#CDCMfAwR)EVecHJ$!^qXo{W1(*R1 zFs&)JNl;?JI7PUo@5h7NE*Y4L8Z_Mhx+$*}Tg9>M@(cQ}rS~T~0lnf+K{Ge8tNk=l zs>o7#(TS+~*lGXz&qp(f^ZKZrKko==6nT5nHA0?KLyM_NWSt=MBx$VWeJuLW8ic4<`;kSW)`@L2?8_&gQo$Lbvjbio&<|mBltPAhOkuSi^ zAY*LZmY}$Iw}+cu)+-fC?$#q;pU`qTMS*feUk}^$Y#WY3vCHX^Rh9Q@D)*Us`T?ov zk{-{$J$N^A5JJp=Myv8TFdHN>CGaoW^R+uhtx5qiFcAR_Fbe^9kZ06x!aE*%HBGEF zbp036pG0c84$)qcX|pV&JU9eLGJJ#Kpr;h{B*>H2s;159i&n>V3AZRwh=$`i^; zi=&cs@${0}>2jp7acpyUrc1`;g`!%MY|Bqdqea+B_piz3Q+HA30XZ-O8eq1CWZU7< zo*r4EP#;Y8;dAuofCiWc0?zL7j$=$WV9FwKwBoP&Z{KpvB-{5a{m~?X+7@;aLH7XA ze4wP18Zyt9S`r#2YZcjdVA7l&j7~JG1rBP@wWwxS?dJ>FdMZKt-PV4s8@{g$! zW*b!0XE&e$COipq9I3Ry-_AGeE>=fw zu9j#zJak1j`mx}tgYTMoNG@|GtG-I3n069A9%AiUMlrtz z4-Y`Kd`y-GouHBsql`HQ8tr2SG{9^^NNM8BSdvbYb6mu#t4`r2b^lGwMo``mCjBELgmyEfHC-_cU@{?}fD(az_5>HS1qs1&s&oZXtL62Iw+$8~@0Sz$GNcR36vC&SPx&*1>5&Dn7 zLokO6=oR15`${AIIr_&X!^(RO#;Lu)jSXlNvk44P;Kp*`2W=WZa@j+rUA8Ah3_DCf zLztUN|M#!`t1;ztJJ;X=@zN%)`a4bqSIF=9huD3kogL|RIaZj$<@<-(5ZnYvJT!t? zUmz>nsXl|s$78T-UrDGJjlTKI4-Y>CIh=##(tt)W?JoD>=nKAXa~Iq%6`geN(mnRu z?ij^fXAMwuYP9I}4{L^eb{*}v^{LbEoQ|;p`PkwsL8Dv{- zf46w<_^zCi+*YdyRgc&fIeK1H>PfTT(m!7tn#O_1*uLeVLAC5gJ_2_9yN3$XN7297 zhb9KqmyB#C!c$1&5y>q)`eJ5*!2;6@%;FJg=%l=$aBQ>KcnI?;4+3smM%?d+oo5R6 z2)Awm8pTBLgZV6pxpT?v^RHP?8^t%@_2||=e?-$g^LbM6BOlnR>4u+sq~hCR_Dd~m zU^)<5+MardyT1!NkdV1(@VTwN{cQk+9R%?cdU%Ihew6hRZOxmJkj$4nR6PZb@rZZP zD8yM*CL@@6XtXJ<^ZPvEkcJ)$>EjV~9e85Jl76%@#pVi&BORf|1zy+zPZ}xiW!}73 zPy4*_PIO}_So7C;G&)7Xu(jAz!G0XT@7UzhXfefwro{pLPAj=s^2^2bkdOjLqGJtr zG`{1(+Hlz-t=Vv<);jpixYlM6OX>!z`(AOZQDh2felT-4D(@ITvm~qqqk` zjc$PL@Fgwl>hXf}Hg@WUma)m~OdEFOoV)^h#kXgOs+0H)?iqFQcVCjRZP;?gx~BJ) z5YKIsm;sGq+O4}?c}#FTJazrJr7NG^BNhGea8cFUH{(8s05hOb%ts)Hw~~BvKOiGN z<&6zv-Z?9k3p1cmOvK-quVC+d*@~YZiJ794ZYbB+t$;=`TLSZgB%j=qnxvv9dOm*g zxh)V35!5}PQB0g)G3(eWI(h1r$kD?lv~8O{Di>xzqnPNCrZw8sue;3eq6PQ<*1q0a z^s9i+fJQN0sG5)I+VlWiGlrmzAj&MDQOw7{XWyhQN#32fjVPT4SsDyV<}o7WBOa!# zG;L_VrMSApP{=}7`vu=?d7q=agIU(~@Qz1piD?A-Psfw4y0qg0TOJh`Th8rZ2@5P~ z9rTJ1RWe`K*_OOc(UD}-n=x~=x6mkNUxG7zKwoIwrX6pFklnd{R`G}dA2820906e# z=4fqYr-CoL;&Y zC4i(Bb4vLei^VCB4zhAgIg5NFle>8VyCD?GDVu&&V~36_L#cv}T!%&uZ0hN_Zv&1A z9o##5xxg{bq1BCq-jRoxJTz#?=EQ_mpxPFtysui!4n?6C!z71dA_)hkb4lE!P98c# zST}OyPDei22NMuGhC2VixB0r&A^TvaCG+ciUv}6i%_j!}w6!v@{Mzt?7T~})jXJFr z+fOUVNiG8#mBC~ZX_l+E?sQ1HGjXLEDD(xop^`J(Fy|nxgC3<^2{<^EaWq=`1t>fo zQKOgd$n%&P!=H9csT{#jt+5%svU(WRfR9i584I(^fkJ&&Q- zC3C)&Mm`$Fya0Uu6ZF1>IaqEOO+=e?=yGgpbnS0oeEo(EXn;AAxJ$1C<{_Z>F?ms@ zv4l9cC(Ihu#qaLA5Y8&`VUhWhEuXU}@{6uzFKb{rK``f#Flw93JfXTcG9?u-x^AVH z6@|_z%OaO~il~X?Qm~4L&Kh)zLAbC~<{1L7Z5}5TwYDk;%X)f+E_r{Rp`8&U0Q(o4 ze9lB^C1u^72`s6@iN+qk(8aPyENqT&9A43&PzG6#F%4NcZI{7AqZ#93>)CE)i|vSd zjUpOSBy57)N3`LmfA{S0+`g&kg|DZ)y8rjHsDU^NC?AbBdT(G(v|eGlB{1=b7G3gr zuge~zIOMiBoHVjolNk;(4~=3D1LjxIfI=%77SC(LPu=I_KMqMn+uYIe)CX=ohcM&1 zI}rvnz_c5ebAfpRJ+hRO&F~JXsMj%rH%`AAbq9mg($4S&G>X}ZNKI?d`wFiE#}Vf& zy|wy;*><)89@1F-#*vDO3Uaj4be`4@ezfA1RbKGDmUlT+Wd|OQ9e)p&SHCj;cslp9 z%lzTHq~?@S9fHl*fJQJaPo_I`$ghLW0!n}wNHP9i9SRm;Fs#0yESQtAMc%79`h4o8CzaO>L`ti#qrcA&FS1sXp+IF## zg*{k*9NW6AwdnJ225r6>p?dRBeZL6;8m-ziG{L1TF1HvYi}ZT!6*U24+Lc;7Vyj5k zU9cBckTToy_~lp=|63h#Ep{*@8Z311eR8z$TK(!n`@VSE!BmwP({;gsMlm;&Wi)?S z^XQor_2ve@-+>N097vdlfW|jF0gYlJtAAqa-i?#NyoA=Lq0FIo?#*2k+J^If0(##X z$W}(|s8K+pNZyGcpi#`PAyynw$--nlJEfwvov&N@=X@+U2eGGe4R5sR-7h5(xOZ4 z{Pi#`)(!2fL|)Xe)RrTHYizgl%#AFLL_M}@x#WPCaZ6oL+ki$fdl36(0xOa04=TaI z<7O{A;}Iz4No+py&?x4SzNQQrQRd_WKUu2g$-8nIj3&XiF) z;kjkSRaJC_4V@+-CxY6~4O7}3s zPD}VGv}cAl&mMo#yWh3+^gEm_1T>0igI)qZRI?X6K5gE&(OtOCJn%yU8pX6OHP=0j zz*YN*ZfLyh{r3)|Bim1ZH|*@^PsSQjTz7YxoMZWSOhvK|Doa<0XG-G{6)&1Qv;WYs zv=r55>6cqSG|`IO9uKq2ukBgn2cXsQh*#NYr}%}-T8(_`s~ReHn?l=+d1z1-yB=>b zWAStWCLZxXLQ6fe=94j3F=iebVBP?T6~rQ)YN)8Bvzj&K<K+mHqIThr0m1gbP`Rl zts2lM=2FzSrwkZP%H1eqUx%`+JmUr zU?(e}k*cJhB+S}obXSQft}0JYqZD@jjD!cA@8wE~@9ETDZ>n!rl>!>Y{u%uCP4deZ zsP~p#@|}p^uUUhfu?T1svpK<;Z$R(!sb{r#AR0-SflUi&6tkExYnverXqK7 zpG#hH`2X^gI&q4f0lyzZW`aZC)&2pEVxrfXPnb_9PfcZV%z_ac zTJ=OX3D&&Ytlj<2uWzIELQW(yE@eJXs)z;v$w_`K7Mq#1ZDkjxh7wX0(;Zc`WL;=e zkZ>N7V0j`F<|+wjw1^K;v%Tq&g{hsp_g*jVJnF99sb;}uOhBWUBgs~nkCUycUM~;x zV+iwol;>+|d?iesSqR0vcfM z09$9h>g$pkEh#uPDv4sYl1&!S0CPX8O?nqtim@!uF7zMH#`NcjXBGb1fV?!`3WJw+ zIM+Q7jbiQz%p(}nX|&|Lh&ueSKc z@KBc|F;$l&Rx%Vh+k&Pkjln~)j}TBAX)Mg6fb_F>i&G!j(8FW9)A6&JEONJi63_tC zHY2hom`PlfTkwvDrrPipCIUk$L1pn-HCir0k0XtA^SQ(<=rcP}vBRY~Eb4Jhh<1Oa3aA4Zs*kf-#5lR)N8BlkO$7SO(Z{FQ^QKK=mf1=wQkS@@1ztdkC_-sU3BYpKQNq0y=!KE)PaZaN>1Zk|ccV>DuTN>*@OU4$EP+Xid}Fh!B7xkmN5@#v|4tF zZYt<~S7PO|Yf5GwNM{0vPU_Kh!TmUd8aU*t=LqxK`rtQGrkIO83mqSiNMoH2d~xif zH=o|grn7*)dD8cbl^z!HP)$*lVUR#Tqg6QwEsCYNtg3wXQo+*?p1pr6TD)6}^S&rS zRRZw@G>W+hm^jCrg(*WvwATp_AA42L<+(&2&?x3Tz^j+m-w$v9 zvDxITy=v8Z)v8rhtH^&oLb?e1T67)XwEqFX*IO3!4){g)k}s?Ue&O~I`-B z`HapCM(K3#!A9LW8fuDDig{o;U&0!049V!sU<~KkB=#IU8Z=AJOd}7v^%L8jf6S&l zO^}`b&R`6uKTfcVjDyiZNW=pNapjtI3-9|P2agPA2BSEupMave-?tR#lt2gb!+Ev4 zcHMvG8O?BJFotsqajq=l)M`(arC&X9>aX6w@@GzG24gtsbnBaUbagc?)|9P*Ho=|2 z8165U-psQ|$O0$1%ebxY9C|!GOSYvcgCWjssP@k?{CZA!5NkEH=s|a_g@O?mjYIcK zU<{`Zzg`#l$IOXDiMZoMJII8s4~|)^r#`5^T4CO!xlkmw(v7AG@5!jg@{zGAsw6E?-#@L2P4@M#4tvD7J(< zS{ltu*o*6(f4Sn+S34$hKNEM|g0c$;&xaM037Zaww<))tC$~2D+dgnG73H@Zn5L<# zuqo+PADwzhw~`ZWV(F1|p%}-sHLNvBm%y-lp0TLQZ=N%1hOt9mOc2*VDKs@-0&n;! zIJdWW`zSQs>lcdS;P{}@!}3)178GEuHi|j>!dmbJt@e`^BJ=P0+UVVHT!!7_S8egt z%T9U)m6E`H-(z3=W~*7cX(0U&xnrTT1jgu$f=}-W5oN{k z7&)Z#H-EL;r)bbVjLyQ=4Blw=E@yiX-axaR6&iKlTpYZ55`4&L&R`7Zmng5X!rRzK zn<1VxWyKE$`hXLvL7JD$_}y;frE4)uZ7WUcpAe&xrD-xcgE5@Gl;DVV@kvQ4 zJ1uuFx|8gAuoliFS$u@fCotB+Y{G;%Ey4>UlAS0N2zy!m69uwGJ1?tlUxZ3`y=R~+ zB_CQtT+R%IQIp_JEEOArbcG$sXwJIRfG~qGoSOj@J8oG0xU+XbAFnQ`u^B#W%{8n% zpopVNiU+9cV^D;mdp_1ydvQLytNTpc8PeA$p1%Z#HSG^!|BFaF!rQo})9ad>Oy`u| z{hPrU&TC2gcknq@*Ck-Uv|0?=o z+Rv@Y*sSNT0Bc7ZCEr*ZeoymZqA6DbqwKWM74YDuw}w;)@M8l`>#DxE1!x~2F<)oy zNTuZt5#5-|miVwB#8%+kQGz8hNPU?lgH&=CgQso^#Mi)2(_lRgVtu_O;U^Ee$VnC> zQbms2tA)xE7~=FBz8An+)kDQpiVdCOsu4}NH2?&ivFFN@klx#>&AlriQ3=KEyww^a z!jJj1nK?SwCSr?*CMa=$My!&@E9VQ&5*TC8#;`;8F>*i6%bI#xXy3)RjNzO>oNo!vA`KW6BAuB&3Y`fIasG_x{=~evu4L9Yn^w{0N7Dq(>%#2x1gtr-tNg*(6S!Xq?r}p{Q7(??jcbc61$0 zdw0i&jl1^Zju5Ml=9f`tJ@zG|??)5p@k8j8R7RDZr?{T1XYMud0HY^!O!w-wCz(6a zSLm=#R9?>le5{lyt0Cg;ssP_)-IzrE;+-c5b} z)i-+%c&}xez34)*&?gb1m!A$Af1Kz+DR4Y2AoVsT%qI34@-8~fNAV_lK@;BM!7eTDHjMyLx-eCbr_L!?PG zE8Spt5?PT>f-=)(%YM|n0z*HdArv9UK7|H{sDp4aGz>`OQFOU*3F^;tuTgMTxH5g6 z41`UvXiwb6(#Il*O-$}#YGv)*f0%dfoZmapH4$mV`F+@>OQ+lc8P~VN@eAx|Sy1ii z#+$El%RBYv;y4gk7lV$B-E=>`E;-lp>LYVBkXFg(l-4wS>3&1+HL4xe73X6Qq*peG zuABQlK|o{|Hr1rEg$9zQl;g#2oii_V*6B&cY= z6JcH$%|=a>ek0~qq2mf`kd4ljz!=UGi1RqYr#Qs|u=c9OM4B8|&R`7ZT;klNNT7`C+fnF@EI0VCOmCU=*d33E1Vvynp&DFaE-Y;hp)}X8cS~M8*zF&0y>^ zeQ@|jV{30c-{yEuwZVP_4f2P zs+6b*NIUP(Z~ov3!D`hySFjcxAKqUg-YK6-uEMoVA*1w@xQpbW-D$Aejg<2+F-)GwNY zthpW7d@8LB#!ypSZibK>1!a-M!W%y}csT@B%e`&}V>o|FoLtF+mBj!{^AMHShivkz zyT7y=IC+CzgY+{P!%5dwA2GLgmbfJIC8_gvZ`{5tI<;NQ#M#_e$490N2L07nq+{4g6u)voy80hobloBo)ySlY<@Bf%~ z^94U*BN^ctj5V?);}f0|Z}86ozfoJIul9xe7q9u9Ph@%xB~h-s$e0Y?@ZSae^%h(s zf!imV3iOc)H(z-F*=xb>Y~HJx--@p?|NP(D+^cYH4Z0}}!(8V*HD#~m=hvety$-a*$Xy|;@_b*{CPbv?E3;p*55GWrio%b8*hSnwu@AS+bo5Wa^aABqCN{F8AL;f9eI1?iF#PsU44DQRgE+SZ zs1ejL5gG6Wy**^n3U)JyFAgu+aI_&R0j^vNhD%Xxk1`$MM9WM|ee~{5g=zcqKRC~W z!lzY4qYdgWbYOw%prdF1_~tj@BXiZvU?85gXa9d=Whipl-!N0So{Bfz8=lMok>}}R z*QS!)d{{7GFZ7T8Rw1AJ!7;&w1BlAQ98$i|`I%U)`wJ4VP~0fRrYv7b#pS8ZaI!w$ zX%(%viAVTnYI$kh{>Z9gd_Ct)N*KZOP!S#vDjKr8L z;n9Vipabp{iAo}fIpJe=-DL8pGuHSK%Jb4&U?Oz=>`Cg(3-EBX+Uz*8Y=kx2G-}Siw$_H>Yxh@5KjpL8#omvnNxdGvGluHIr{N6yP(yFJE>1l67Ne~tcz5BggTEX<1n_ZM$21A^F*tm+rW2jLX z1(Pe@^#Ohax{nL9xR%(@Crd$;d}r2wLacDfpwlrmLPfw=Qs*S(tz02z$oWGTt|kZO-? zNJ;T#Ji@`7=E|FRSi^e?@opIhRhEBwA1QqspOmYt5>O&YBj9uru+^+3i;%UE$j<9Y zi9vjz{w)N0_Z?E4^2{8Byle)eSzP2<24f`Cv$`_~pW+lbJf`t2FB#8pW-x|x6>*+j z#A$}5@p~;FF)|p#NmuO7EaDUz;OulEtO?ExhB&>mqw{i~6P(e(D0!9Ny>RKR^>?2D zKf~Os1jguWp#wdw=#*_FG<#IFin@`(7|u50dAV z9KH5E(kc8b*dO`1>QN2`LnXuH$AWy^!zt4GMNbYt@olm-`1WX7CVqt>x&E}J-`ePX zbvlzaczvn4TR;O?5T|zflO;Zt1=Z*Z)7hm+XiaNU3n$zEk--?wgQ)r|@qnnJd(}nu zBwv3H)h?X{XE27-Cv?XN&d4iEtUk2;%#{a{pQ_eh0BU!>;4gtU{N8PD)jp@AAGfK7 z3?aympNhmAMz=6)EP*lfbI6kGg#4lwHQ+o=NG^dfoQsL`bi$`z%MyXxjlTAlJCMM{ z(N(saT0{mzoId{__tvHB1p<$oIdrc$M^X=eJiSKv@3B1wIdsW@55^lzoIXzoYk*kAo1SX%VsdET75AV^YB;N>>^+DmL z)?3MQK5U~`<;by1K6cyT(|la-d9r|nXMMC4M=c#ya~FuLBOfvt>z6-gb)v{R!mA^- zQY*OM7j6==E`c$eQFf`UZBvZilv8sT7U|4jOul56`;yQZc_n6z5-+!W^4L?3er6i@ zzgi}QB{0Ovy_3m)&vbvN=Cp!K@1dSp#Gk=iw<3IsU&ui8fB4>iUUUBetX+`Zglsh9?%U5!dHM7< zH&U9NN^9gJ6iUGX_J`M8bm*Kx^SpMlTdTN#u|6!!;QQPAJA3>5;rF=1G#e~x?r)-1 zB{0^lpP&)E%O3OfHoVuWeTrs(q^Sl%lC@S+M3ZF@0@t@?+5nds})t8#vr%0#%WH5$v9njSW&+A^&j-CIhQpzVu5C<2+rc+Wg7(>gG zDQ>T#cBo=jKJ)k{JDop@-Oq4lFou)oH{8BOoK|w?lPeC{@!pFmrrnAx&0q}Y?v%gx zOKION$$Q~WS01~=jl}s}5oZQNoV!y(+$gKn9pd&&;Z-s$*W3i*X9qV>tcBun!OCQu6aNlzUM7r}+VxD;V3c~-N$Ll;BN+F@$4@fB)Z9>Tufb?D7(@Fo8T}qdvzDZI zzJ_F0)|2sPl{!AgV!5=~eg?1jSFHirdb4;GlKH#C@16GZ%RhIR4;pSF=r!!-@OCK! zJq3IJdwW>1BW*1E*!b>Co$mn*4PH$OUgD;K{~X?b#M6tXfbW^J%wUMq2y$N&59EV& zKEf@3kfqeyN9`;y$iw?9yPqj)r3Ocp&%Sop%1hf0Z&UD}lc@nb%G4Z(?u>BlGKr5M zB2vtp<qp#+9F=`I~}xT;qX-c(gz%)LhVlx78MNX~fU ziq_JU#gcdCf%c#ekp^S2!WbXkgqU-h&35#q=1yO)eJPCLe3v*oD2ovu>9><3zy2AaF ztSHT#4Tvp)F`OSK&Yu;Xffx|Z*y+AB8>)(in7|m$J~Ch=_QKh)aLAHlI{Ze;C}$A` zB4ZgPUV|NH&OU>&ChlMpqZ^wzt{1EHQuj6X+s7niBDg0oZZKgH7& zdyxc=M&Vx0wcntnq~Xbb(5m|wa7E3g@j7fh}I+fY}OmL zKd(5{UnA2uud{zrgNqaYjdh=kPv7#m?dGylV#f0fhNx-N)xq=`egsTyTKpD@z-XQ8 zCN`bHL~_9tY`g8~N6oKA?KpnCpaV7C&aCzqoU>>`5-3xQ)?s&5bGan&)}o)IE*#|L zQj{ak7K{5qNxp!;dKr3MA2tg|NV;&Kp}VKkE$M5#OG9W&zGYiJ2D_dd3#0d^`OZgJ zbVI2$Ie~^EJvxHLu*J${hyjjobKH~JVAB@LNrqyW4;p;4+=t-qV4A8$iYfZ*Y2}JL z=#0LYql~9aEZLzi>i44AOK%g6cT0t`*g;7K;$jnh$@?c-mhAS=`)31zz8G64LE80A zS(o?Np84FI7n>I9kekpKn|MUtiYGv#N?&q+sea!E0BidJbhbXMd-iNzRI#}GgpSHI zzq`MvN)4&;V810WhChbwExy2me^lh;KJF=$(?6)N0u`edOXgo`cz>8ll!*? z@Fbp0ot{mXMJyxRbhZ{LRxfPzVZ?vLoO#3iAdRCH+CI~^R-KQZ z>YJgh!~D>Q#{7*beAp>(dqH7D*cJwfIDc2)X%iNAxqw7wh_6d>O8FF4EGQXGRYjww z3rLda)GoO1Lu=|S=)w`ra+nQ%@_R+v1Zx6gSp6ErQo`rK2u`%OgxfD_t`n>ROJaX* zqNOl~vm2o57b3iQKJh>65U6H0oWU5*KKg~AfKha2(E8Fubhb>SFS+P4_N-5I2?ul0 zX=L1Re$p$gFQfI{L7vH;c&dJwpyCdD;UuyTY8erJP@<=zU|3Sn7oGG)en>|ULMck) z-$k7KUXsX}UNN}zBnUAQB77?IoKfq^!5E_+^CgBn7ifd-G^$aH+lS=v5ynBR42tYG z_FRCl!%qAs@Y)d7o7(9Mn06idmAI&M^vWpy$DGPbJ$?_SXkGOEaGcdmK!1G%CRjSF zI%LTSJlI{%)hxSXUQ}O9u>Qc^kXbk!c|u2}!HA{niMMHDa2br@J%@yE)<1ybg+Scs zo-a$}JZdcjr5dS*Rsv(FeHGu}C8CO{!?DXFYKpT2 z#&C9n5pD~a1@v)7z(=s0)fyG8hBph*X$g$s`~lT}M{x*6-`~*pa2a_CjNm4AxhqLws{I*^ zwZLRqeH3g53V3u!AL&zC8x&mz=NB}M`%08W9DOnvYlVDpx2~6O>eWrY5vS>@!Wd3& zGN=#6SPilpNVQM&g$~z?!)FG6P-KpRnH;SN?$fDngWXEU`f?E+^N)3h%7I@r7(-qa zo5|1mQ37L){5KmxEdsyUzsu54^ajPxlfY-aO(ZaE;z@e>r}zjp@M63b?0k2KD;zH^*MLA|L#Jdsjy8Y-^YCoD^6tN1GwEwf_r+zNr~P*9El>AA+F?}nq!BQOVOKoPmnfAi zN2h~txKeJrh%-8c42C$VLDiW1PeW~+`{~hZOA$w$8H~~C_pgTer0+Y`BV@3TY^xkR zbH`8rVM-t9%<_|(y$aiyW*}DqmNbM~Hy(7}vS~LR&{p}xi`PB1>+eo(Q+x3VRL^Cp zul(t@%7HH)d+~qt;AY2cDOTb-tQ>AW`PxOR+GqLvnrFym6x_Fpo>%e**yeK2PUpN= zTm2KCLi{vnbJGd$_gkdnM+R>Ue}wQ~75v^L_^@o1Z3#1mm%tcKmSuLEWB6`x0iVBm z;fl$Jj|QhkJuvzUW0?@Xy66Hj7~&jCJv)${+Z=De8FSMuT4hzw2SbP=p{N*^{?jc? zZjyi$gQb^rZ}{upZhW&FLn+-y@IbQ^rhnKt5xoS+>?i1#(RIdcKE;fD26Uz5GwU6z zQ~Onn_A`22S39b2R(cl5r4+@EB-ekgx3dWhoA@0W=&P&zK^W+*z+1OjqIe)&w{m@e zXNblfaJY~RH#&FqF5ngkbf@H7Ys-gscZ*U*coT%)vS4lb^JYWXOtFpZ8*78#xStjc zLp&fPdDZwA_Z;~x=DV|TCW9f)SBTClR)A;*t7ldXuv5ouW%26-jUiw>KP0vHuvL}iy$JSj0VkW z*PNr0r@kqJF`N}@=OV(VeHZ3qVPmJ^e>|#w!rjdbErTIWKT7#ae)lT&UA#H2k=68g zlfGv#hBJ{+6yYU&!Aj1|18#oq zn#IH!%5nK!gnWd{b8wg6$}_I`#hKGld~63{&lsZQ!}6Pz?~dE;wR3NKJnq3|y45Ws znJo#K;sOzz44ZjrdzcAdx0!wwx?eHpe6cz^cc3c?>R zG{<8!*;LkXDh_VOZi&>BOoH2j9wr^(Oo*a&@3gut1*C`tQMq`Uhg#jg78|m8it4WQ z^;Sr%KB%u_(HtM06eF9@hPX|NlLqjbv*~}aHXcG;6;TY`sPFmd0 z-lv&mtqSgQpAVy{2A<;o{a;020KPYs#4i&E@E-gqm)eShNKzlxiPte2)`#E_cb5Dz zZt6?T{of+h+Pab99(aF{2L&Wjrw!QrQmI>Y2^!!s18O%wa8c`|KiHa z<~`bNdg|ZUq7>vcs053t-Lvnnna}!>$BR13F4wxm9^fg9s+`%3vP9+`E#iS9L$r^U znyIh7PH@!%#IT3xyYm=*cX>f49KdlIP)JkG7t&{9G^k8xN zaoiMMVH3LY%A9@)jN#;3ojXa?B!=9A6HZ56OA|3>GRiI3B&}-U=8FQy-kOnXH;l5& z!NL0A`5m*m=QFuq{3OxIntMVD zbnaKgnJIA-Ix`r<=|{Q!i#T(i@ImNKK;fyL z+11K`liL+i#mxl6p`Qal^#mA(PS$v6a23`sC8i0^ z490L$Vcg>%E}c|=_gE2U!3#drtnV3&(fMuQOtq)Kb58eyj;PD24jKn_>roSqWH8ne z_0&DNUh0*^COGw{ESRhMi>ZC!kfV&69_Q>SJ4+E zuoG;l4_Y9l4xh%)a>iC7xnv7{6Vilb2+{OKNsHQ{TrD9j12NM4vA+-}MU4o#uoD(`|JIs&6B~Z?CBKJAcanx1gtl&TpDnsA;W<{j z^y(uc{!VJu4~6(RkeVY9#eT={!SbO92NU;{SZWD8@%vH~7Xef!WHA@6orzf^g};yt z42jQJeO%y^)TC<{65^aBGjsQ7kxKN^|5fzm`Y^feh9l6)Z4qPQk@NE1S)eXNfye# zibxn99wSx|wN%9(d^@%Z9ajXAuTz;H-yEdiXgp=Srq*$hs)_S@EV;T#)sDq0aB&4L z3Bh$ays?NeS+U5r_9v5X)fdV57isQxHqA#Yzz~zCucG4^HDeYmbL+V|*;WKGy*FCj z^P+hp$#OnyWU-PA+Alxn>xQPF(I`=KcG9EOT)gmXZ?0ZHLY?I>yE~wK}Q5rBDtEILYVa4vJA#>lG8fXiA?zOF=zbmGk=NGPLE%5;k1=2 zr^C%>R;*ec9Q2enmsy+^5vcSYy5^>Nw;$CcN2~iCDj)yIR^OWX=RldK=l3V`zvp)dmZv|)6U=3)4RXozB+7X+T9=y|9`NFo-mOb$7 z3z1*6JNJS#8zpJGvh>nN(~c4t>q^x5G~a1ckh`upsC)0z9`PG`rlMtkNCFpAuj_`F-fnYMVQdVG zY~;VO^`z;yu9MQw-b{U16kLDwyGEwItap(uvAaBmQ46+!z&$6FMfJ^`mN8^8W0H`h zjPPQ~i7Us=hY&ohy#&T^a;D*i(1)c=HU_FwvSeyh-weiZ`a?1qoys^XU0=)Dlff9y z^GWA=^7t5YG-|Eo(U5c@fiaw4BhD;QB-JKxF^G)z`@<}=#q|uvaK=C&wGUHT3Bp+x zVpLJ?MyD|L)h28DI!1-9DkGz*$7PGu6WUOuC#jn>o55IDs>~kvi#gK(#AuKUm=8c# zvcV;TF`NsCGh?}keOOy%pFbYi`}y6c+CGb_pJ#LxS(U*XeikXwxb0Am;C$H7YA?2g zbRJzKA7PZoIIIJ*#%j?Ah=daRwr0n(*`S@#Gs~EXy{Cc_+Z^4q(v=bz!%5C7t0I|N zVyAN9()+Gn@hDi87r?cMTllmNnk3D}kV58CAnlv$PTFDrCyt1k6fe?E$3JnGI18rN zZZ{eKG8pR$Z`*ZKNJBc?RYu!FU@dpk8I0lNRlM#{!5RCdwqx7sV7$e?XE25{&X3Z* zn}tZo(Cm+=e zX9i<9e?&Tu<-n185L0;(1T(TB$}SErFbjn-oDUJ_zct5c0~R+Ltp{fUV>nk5=d|WH z%``bJS`W?y#&GiTwEC8|`huxRV_wFzP+!V4RbdP}7s;KUCZt+r!SbcwTrv2p_trpK zD@0SHbI4!}=h?)0B;iwwGR@xW{I~Dsn4mjZoJLTN3Q%YyBMlT$oKAxBSzh@{qMc=@nrhkUx>Y8q+y94 zeW|%iW!6;!!>)Lo{tWKR!m0>wQS^ z|Gw92TQM{R4L+uf;!2b3y2ZNk^wPL^D_LW~e=0_Qf;j9v8SPG4F3OsgZ%fue_FmqX? Lyv^YE*yH~JxRaDs From cb431f8b959decd208a314b83df9c37df6f6a467 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 3 Apr 2018 13:27:52 -0700 Subject: [PATCH 118/519] BUG: Adapt to breaking change in google-cloud-bigquery 0.32.0.dev1 (#152) * BUG: Update pandas-gbq to latest version of google-cloud-bigquery There was a breaking change in 0.32.0.dev1 which changed the way configuration for the query job gets loaded. Also, it added the 'description' field to the schema resource, so this change updates the schema comparison logic to account for that. Detect google-cloud-bigquery version for backwards compatibility * DOC: Add verbose deprecation to changelog. * TST: MASTER in CI also builds with g-c-bigquery at MASTER. --- packages/pandas-gbq/.travis.yml | 3 + .../pandas-gbq/ci/requirements-3.6-MASTER.pip | 1 - packages/pandas-gbq/docs/source/changelog.rst | 5 +- packages/pandas-gbq/pandas_gbq/_query.py | 25 +++++ packages/pandas-gbq/pandas_gbq/gbq.py | 93 ++++++++----------- .../pandas_gbq/tests/test__query.py | 57 ++++++++++++ .../pandas-gbq/pandas_gbq/tests/test_gbq.py | 38 ++++++-- packages/pandas-gbq/setup.py | 1 + 8 files changed, 163 insertions(+), 60 deletions(-) create mode 100644 packages/pandas-gbq/pandas_gbq/_query.py create mode 100644 packages/pandas-gbq/pandas_gbq/tests/test__query.py diff --git a/packages/pandas-gbq/.travis.yml b/packages/pandas-gbq/.travis.yml index 92129cc6fee5..423786805e6c 100644 --- a/packages/pandas-gbq/.travis.yml +++ b/packages/pandas-gbq/.travis.yml @@ -28,6 +28,9 @@ install: conda install -q numpy pytz python-dateutil; PRE_WHEELS="https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com"; pip install --pre --upgrade --timeout=60 -f $PRE_WHEELS pandas; + pip install -e 'git+https://github.com/GoogleCloudPlatform/google-cloud-python.git#egg=version_subpkg&subdirectory=api_core'; + pip install -e 'git+https://github.com/GoogleCloudPlatform/google-cloud-python.git#egg=version_subpkg&subdirectory=core'; + pip install -e 'git+https://github.com/GoogleCloudPlatform/google-cloud-python.git#egg=version_subpkg&subdirectory=bigquery'; else conda install -q pandas=$PANDAS; fi diff --git a/packages/pandas-gbq/ci/requirements-3.6-MASTER.pip b/packages/pandas-gbq/ci/requirements-3.6-MASTER.pip index b52f2aeb3289..78f6834f550d 100644 --- a/packages/pandas-gbq/ci/requirements-3.6-MASTER.pip +++ b/packages/pandas-gbq/ci/requirements-3.6-MASTER.pip @@ -1,4 +1,3 @@ google-auth google-auth-oauthlib mock -google-cloud-bigquery diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index eca6f9ce664d..94f2c42454b0 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,9 +1,12 @@ Changelog ========= -0.3.2 / [TBD] +0.4.0 / [TBD] ------------------ - Fix bug with querying for an array of floats (:issue:`123`) +- Fix bug with integer columns on Windows. Explicitly use 64bit integers when converting from BQ types. (:issue:`119`) +- Fix bug caused by breaking change the way ``google-cloud-python`` version 0.32.0+ handles additional configuration argument to ``read_gbq``. (:issue:`152`) +- **Deprecates** the ``verbose`` parameter. Messages use the logging module instead of printing progress directly to standard output. (:issue:`12`) 0.3.1 / 2018-02-13 ------------------ diff --git a/packages/pandas-gbq/pandas_gbq/_query.py b/packages/pandas-gbq/pandas_gbq/_query.py new file mode 100644 index 000000000000..864bbb3749d6 --- /dev/null +++ b/packages/pandas-gbq/pandas_gbq/_query.py @@ -0,0 +1,25 @@ + +import pkg_resources +from google.cloud import bigquery + + +# Version with query config breaking change. +BIGQUERY_CONFIG_VERSION = pkg_resources.parse_version('0.32.0.dev1') + + +def query_config_old_version(resource): + # Verify that we got a query resource. In newer versions of + # google-cloud-bigquery enough of the configuration is passed on to the + # backend that we can expect a backend validation error instead. + if len(resource) != 1: + raise ValueError("Only one job type must be specified, but " + "given {}".format(','.join(resource.keys()))) + if 'query' not in resource: + raise ValueError("Only 'query' job type is supported") + return bigquery.QueryJobConfig.from_api_repr(resource['query']) + + +def query_config(resource, installed_version): + if installed_version < BIGQUERY_CONFIG_VERSION: + return query_config_old_version(resource) + return bigquery.QueryJobConfig.from_api_repr(resource) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 6d5aacf85106..cca6d2119568 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -4,7 +4,6 @@ import time import warnings from datetime import datetime -from distutils.version import StrictVersion from time import sleep import numpy as np @@ -14,7 +13,11 @@ logger = logging.getLogger(__name__) +BIGQUERY_INSTALLED_VERSION = None + + def _check_google_client_version(): + global BIGQUERY_INSTALLED_VERSION try: import pkg_resources @@ -23,17 +26,15 @@ def _check_google_client_version(): raise ImportError('Could not import pkg_resources (setuptools).') # https://github.com/GoogleCloudPlatform/google-cloud-python/blob/master/bigquery/CHANGELOG.md - bigquery_client_minimum_version = '0.29.0' - - _BIGQUERY_CLIENT_VERSION = pkg_resources.get_distribution( - 'google-cloud-bigquery').version + bigquery_minimum_version = pkg_resources.parse_version('0.29.0') + BIGQUERY_INSTALLED_VERSION = pkg_resources.get_distribution( + 'google-cloud-bigquery').parsed_version - if (StrictVersion(_BIGQUERY_CLIENT_VERSION) < - StrictVersion(bigquery_client_minimum_version)): - raise ImportError('pandas-gbq requires google-cloud-bigquery >= {0}, ' - 'current version {1}' - .format(bigquery_client_minimum_version, - _BIGQUERY_CLIENT_VERSION)) + if BIGQUERY_INSTALLED_VERSION < bigquery_minimum_version: + raise ImportError( + 'pandas-gbq requires google-cloud-bigquery >= {0}, ' + 'current version {1}'.format( + bigquery_minimum_version, BIGQUERY_INSTALLED_VERSION)) def _test_google_api_imports(): @@ -447,8 +448,8 @@ def process_http_error(ex): def run_query(self, query, **kwargs): from google.auth.exceptions import RefreshError - from google.cloud.bigquery import QueryJobConfig from concurrent.futures import TimeoutError + from pandas_gbq import _query job_config = { 'query': { @@ -459,29 +460,23 @@ def run_query(self, query, **kwargs): } config = kwargs.get('configuration') if config is not None: - if len(config) != 1: - raise ValueError("Only one job type must be specified, but " - "given {}".format(','.join(config.keys()))) - if 'query' in config: - if 'query' in config['query']: - if query is not None: - raise ValueError("Query statement can't be specified " - "inside config while it is specified " - "as parameter") - query = config['query']['query'] - del config['query']['query'] - - job_config['query'].update(config['query']) - else: - raise ValueError("Only 'query' job type is supported") + job_config.update(config) + + if 'query' in config and 'query' in config['query']: + if query is not None: + raise ValueError("Query statement can't be specified " + "inside config while it is specified " + "as parameter") + query = config['query'].pop('query') self._start_timer() - try: + try: logger.info('Requesting query... ') query_reply = self.client.query( query, - job_config=QueryJobConfig.from_api_repr(job_config['query'])) + job_config=_query.query_config( + job_config, BIGQUERY_INSTALLED_VERSION)) logger.info('ok.\nQuery running...') except (RefreshError, ValueError): if self.private_key: @@ -598,6 +593,15 @@ def schema(self, dataset_id, table_id): except self.http_error as ex: self.process_http_error(ex) + def _clean_schema_fields(self, fields): + """Return a sanitized version of the schema for comparisons.""" + fields_sorted = sorted(fields, key=lambda field: field['name']) + # Ignore mode and description when comparing schemas. + return [ + {'name': field['name'], 'type': field['type']} + for field in fields_sorted + ] + def verify_schema(self, dataset_id, table_id, schema): """Indicate whether schemas match exactly @@ -621,17 +625,9 @@ def verify_schema(self, dataset_id, table_id, schema): Whether the schemas match """ - fields_remote = sorted(self.schema(dataset_id, table_id), - key=lambda x: x['name']) - fields_local = sorted(schema['fields'], key=lambda x: x['name']) - - # Ignore mode when comparing schemas. - for field in fields_local: - if 'mode' in field: - del field['mode'] - for field in fields_remote: - if 'mode' in field: - del field['mode'] + fields_remote = self._clean_schema_fields( + self.schema(dataset_id, table_id)) + fields_local = self._clean_schema_fields(schema['fields']) return fields_remote == fields_local @@ -658,16 +654,9 @@ def schema_is_subset(self, dataset_id, table_id, schema): Whether the passed schema is a subset """ - fields_remote = self.schema(dataset_id, table_id) - fields_local = schema['fields'] - - # Ignore mode when comparing schemas. - for field in fields_local: - if 'mode' in field: - del field['mode'] - for field in fields_remote: - if 'mode' in field: - del field['mode'] + fields_remote = self._clean_schema_fields( + self.schema(dataset_id, table_id)) + fields_local = self._clean_schema_fields(schema['fields']) return all(field in fields_remote for field in fields_local) @@ -709,7 +698,7 @@ def _parse_data(schema, rows): col_names = [str(field['name']) for field in fields] col_dtypes = [ dtype_map.get(field['type'].upper(), object) - if field['mode'] != 'repeated' + if field['mode'].lower() != 'repeated' else object for field in fields ] @@ -847,7 +836,7 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, for field in schema['fields']: if field['type'].upper() in type_map and \ final_df[field['name']].notnull().all() and \ - field['mode'] != 'repeated': + field['mode'].lower() != 'repeated': final_df[field['name']] = \ final_df[field['name']].astype(type_map[field['type'].upper()]) diff --git a/packages/pandas-gbq/pandas_gbq/tests/test__query.py b/packages/pandas-gbq/pandas_gbq/tests/test__query.py new file mode 100644 index 000000000000..43ab00f3171d --- /dev/null +++ b/packages/pandas-gbq/pandas_gbq/tests/test__query.py @@ -0,0 +1,57 @@ + +import pkg_resources + +import mock + + +@mock.patch('google.cloud.bigquery.QueryJobConfig') +def test_query_config_w_old_bq_version(mock_config): + from pandas_gbq._query import query_config + + old_version = pkg_resources.parse_version('0.29.0') + query_config({'query': {'useLegacySql': False}}, old_version) + mock_config.from_api_repr.assert_called_once_with({'useLegacySql': False}) + + +@mock.patch('google.cloud.bigquery.QueryJobConfig') +def test_query_config_w_dev_bq_version(mock_config): + from pandas_gbq._query import query_config + + dev_version = pkg_resources.parse_version('0.32.0.dev1') + query_config( + { + 'query': { + 'useLegacySql': False, + }, + 'labels': {'key': 'value'}, + }, + dev_version) + mock_config.from_api_repr.assert_called_once_with( + { + 'query': { + 'useLegacySql': False, + }, + 'labels': {'key': 'value'}, + }) + + +@mock.patch('google.cloud.bigquery.QueryJobConfig') +def test_query_config_w_new_bq_version(mock_config): + from pandas_gbq._query import query_config + + dev_version = pkg_resources.parse_version('1.0.0') + query_config( + { + 'query': { + 'useLegacySql': False, + }, + 'labels': {'key': 'value'}, + }, + dev_version) + mock_config.from_api_repr.assert_called_once_with( + { + 'query': { + 'useLegacySql': False, + }, + 'labels': {'key': 'value'}, + }) diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py index 2df1b9bde935..abb670ef4c53 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -1266,16 +1266,42 @@ def test_retrieve_schema(self): test_id = "15" test_schema = { 'fields': [ - {'name': 'A', 'type': 'FLOAT', 'mode': 'NULLABLE'}, - {'name': 'B', 'type': 'FLOAT', 'mode': 'NULLABLE'}, - {'name': 'C', 'type': 'STRING', 'mode': 'NULLABLE'}, - {'name': 'D', 'type': 'TIMESTAMP', 'mode': 'NULLABLE'} + { + 'name': 'A', + 'type': 'FLOAT', + 'mode': 'NULLABLE', + 'description': None, + }, + { + 'name': 'B', + 'type': 'FLOAT', + 'mode': 'NULLABLE', + 'description': None, + }, + { + 'name': 'C', + 'type': 'STRING', + 'mode': 'NULLABLE', + 'description': None, + }, + { + 'name': 'D', + 'type': 'TIMESTAMP', + 'mode': 'NULLABLE', + 'description': None, + }, ] } self.table.create(TABLE_ID + test_id, test_schema) - actual = self.sut.schema(self.dataset_prefix + "1", TABLE_ID + test_id) - expected = test_schema['fields'] + actual = self.sut._clean_schema_fields( + self.sut.schema(self.dataset_prefix + "1", TABLE_ID + test_id)) + expected = [ + {'name': 'A', 'type': 'FLOAT'}, + {'name': 'B', 'type': 'FLOAT'}, + {'name': 'C', 'type': 'STRING'}, + {'name': 'D', 'type': 'TIMESTAMP'}, + ] assert expected == actual, 'Expected schema used to create table' def test_schema_is_subset_passes_if_subset(self): diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index d3c353118b0b..ebe147c30e69 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -17,6 +17,7 @@ def readme(): INSTALL_REQUIRES = [ + 'setuptools', 'pandas', 'google-auth', 'google-auth-oauthlib', From b92a73a9d9ccdb148aecbca2cf2b371ab5b6d27d Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 3 Apr 2018 14:42:41 -0700 Subject: [PATCH 119/519] DOC: Changelog for 0.4.0 (#155) Preparing for a release of 0.4.0. --- packages/pandas-gbq/docs/source/changelog.rst | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 94f2c42454b0..30281c3d0345 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,12 +1,22 @@ Changelog ========= -0.4.0 / [TBD] +0.4.0 / 2018-04-03 ------------------ -- Fix bug with querying for an array of floats (:issue:`123`) -- Fix bug with integer columns on Windows. Explicitly use 64bit integers when converting from BQ types. (:issue:`119`) -- Fix bug caused by breaking change the way ``google-cloud-python`` version 0.32.0+ handles additional configuration argument to ``read_gbq``. (:issue:`152`) -- **Deprecates** the ``verbose`` parameter. Messages use the logging module instead of printing progress directly to standard output. (:issue:`12`) + +- Fix bug in `read_gbq` when building a dataframe with integer columns + on Windows. Explicitly use 64bit integers when converting from BQ types. + (:issue:`119`) +- Fix bug in `read_gbq` when querying for an array of floats (:issue:`123`) +- Fix bug in `read_gbq` with configuration argument. Updates `read_gbq` to + account for breaking change in the way ``google-cloud-python`` version + 0.32.0+ handles query configuration API representation. (:issue:`152`) +- Fix bug in `to_gbq` where seconds were discarded in timestamp columns. + (:issue:`148`) +- Fix bug in `to_gbq` when supplying a user-defined schema (:issue:`150`) +- **Deprecate** the ``verbose`` parameter in `read_gbq` and `to_gbq`. + Messages use the logging module instead of printing progress directly to + standard output. (:issue:`12`) 0.3.1 / 2018-02-13 ------------------ From a606cb174e574cf08ef68bb85bd4db15f732db3b Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 3 Apr 2018 14:59:08 -0700 Subject: [PATCH 120/519] DOC: restore build instructions. --- packages/pandas-gbq/release-procedure.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/release-procedure.md b/packages/pandas-gbq/release-procedure.md index da4b011ae670..3b3384b98515 100644 --- a/packages/pandas-gbq/release-procedure.md +++ b/packages/pandas-gbq/release-procedure.md @@ -8,7 +8,8 @@ * Build the package - twine upload dist/* + git clean -xfd + python setup.py register sdist bdist_wheel --universal * Upload to test PyPI From 2704504302e8ac99c7a7fc26731cdef5d78ed30c Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 6 Apr 2018 11:29:39 -0700 Subject: [PATCH 121/519] BUG: only show verbose warning for new pandas versions. (#158) * BUG: only show verbose warning for new pandas versions. * TST: add unit tests for verbose deprecation warning --- packages/pandas-gbq/docs/source/changelog.rst | 6 + packages/pandas-gbq/pandas_gbq/gbq.py | 20 ++- .../pandas-gbq/pandas_gbq/tests/test_gbq.py | 147 +++++++++++++++++- 3 files changed, 167 insertions(+), 6 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 30281c3d0345..ffa62f1a7a4a 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,6 +1,12 @@ Changelog ========= +0.4.1 / 2018-04-05 +------------------ + +- Only show ``verbose`` deprecation warning if Pandas version does not + populate it. (:issue:`157`) + 0.4.0 / 2018-04-03 ------------------ diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index cca6d2119568..f8bbaf22d7ec 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -14,10 +14,11 @@ BIGQUERY_INSTALLED_VERSION = None +SHOW_VERBOSE_DEPRECATION = False def _check_google_client_version(): - global BIGQUERY_INSTALLED_VERSION + global BIGQUERY_INSTALLED_VERSION, SHOW_VERBOSE_DEPRECATION try: import pkg_resources @@ -36,6 +37,14 @@ def _check_google_client_version(): 'current version {1}'.format( bigquery_minimum_version, BIGQUERY_INSTALLED_VERSION)) + # Add check for Pandas version before showing deprecation warning. + # https://github.com/pydata/pandas-gbq/issues/157 + pandas_installed_version = pkg_resources.get_distribution( + 'pandas').parsed_version + pandas_version_wo_verbosity = pkg_resources.parse_version('0.23.0') + SHOW_VERBOSE_DEPRECATION = ( + pandas_installed_version >= pandas_version_wo_verbosity) + def _test_google_api_imports(): @@ -791,14 +800,15 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, DataFrame representing results of query """ - if verbose is not None: + + _test_google_api_imports() + + if verbose is not None and SHOW_VERBOSE_DEPRECATION: warnings.warn( "verbose is deprecated and will be removed in " "a future version. Set logging level in order to vary " "verbosity", FutureWarning, stacklevel=1) - _test_google_api_imports() - if not project_id: raise TypeError("Missing required parameter: project_id") @@ -920,7 +930,7 @@ def to_gbq(dataframe, destination_table, project_id, chunksize=None, _test_google_api_imports() - if verbose is not None: + if verbose is not None and SHOW_VERBOSE_DEPRECATION: warnings.warn( "verbose is deprecated and will be removed in " "a future version. Set logging level in order to vary " diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py index abb670ef4c53..93c48b0c1ac9 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py @@ -282,8 +282,40 @@ def test_get_user_account_credentials_returns_credentials(self): class TestGBQUnit(object): - def test_should_return_credentials_path_set_by_env_var(self): + @pytest.fixture(autouse=True) + def mock_bigquery_client(self, monkeypatch): + import google.cloud.bigquery + import google.cloud.bigquery.table + mock_client = mock.create_autospec(google.cloud.bigquery.Client) + # Mock out SELECT 1 query results. + mock_query = mock.create_autospec(google.cloud.bigquery.QueryJob) + mock_query.state = 'DONE' + mock_rows = mock.create_autospec( + google.cloud.bigquery.table.RowIterator) + mock_rows.total_rows = 1 + mock_rows.schema = [ + google.cloud.bigquery.SchemaField('_f0', 'INTEGER')] + mock_rows.__iter__.return_value = [(1,)] + mock_query.result.return_value = mock_rows + mock_client.query.return_value = mock_query + monkeypatch.setattr( + gbq.GbqConnector, 'get_client', lambda _: mock_client) + + @pytest.fixture(autouse=True) + def no_auth(self, monkeypatch): + import google.auth.credentials + mock_credentials = mock.create_autospec( + google.auth.credentials.Credentials) + monkeypatch.setattr( + gbq.GbqConnector, + 'get_application_default_credentials', + lambda _: mock_credentials) + monkeypatch.setattr( + gbq.GbqConnector, + 'get_user_account_credentials', + lambda _: mock_credentials) + def test_should_return_credentials_path_set_by_env_var(self): env = {'PANDAS_GBQ_CREDENTIALS_FILE': '/tmp/dummy.dat'} with mock.patch.dict('os.environ', env): assert gbq._get_credentials_file() == '/tmp/dummy.dat' @@ -314,6 +346,75 @@ def test_to_gbq_with_no_project_id_given_should_fail(self): with pytest.raises(TypeError): gbq.to_gbq(DataFrame(), 'dataset.tablename') + def test_to_gbq_with_verbose_new_pandas_warns_deprecation(self): + import pkg_resources + min_bq_version = pkg_resources.parse_version('0.29.0') + pandas_version = pkg_resources.parse_version('0.23.0') + with pytest.warns(FutureWarning), \ + mock.patch( + 'pkg_resources.Distribution.parsed_version', + new_callable=mock.PropertyMock) as mock_version: + mock_version.side_effect = [min_bq_version, pandas_version] + try: + gbq.to_gbq( + DataFrame(), + 'dataset.tablename', + project_id='my-project', + verbose=True) + except gbq.TableCreationError: + pass + + def test_to_gbq_with_not_verbose_new_pandas_warns_deprecation(self): + import pkg_resources + min_bq_version = pkg_resources.parse_version('0.29.0') + pandas_version = pkg_resources.parse_version('0.23.0') + with pytest.warns(FutureWarning), \ + mock.patch( + 'pkg_resources.Distribution.parsed_version', + new_callable=mock.PropertyMock) as mock_version: + mock_version.side_effect = [min_bq_version, pandas_version] + try: + gbq.to_gbq( + DataFrame(), + 'dataset.tablename', + project_id='my-project', + verbose=False) + except gbq.TableCreationError: + pass + + def test_to_gbq_wo_verbose_w_new_pandas_no_warnings(self, recwarn): + import pkg_resources + min_bq_version = pkg_resources.parse_version('0.29.0') + pandas_version = pkg_resources.parse_version('0.23.0') + with mock.patch( + 'pkg_resources.Distribution.parsed_version', + new_callable=mock.PropertyMock) as mock_version: + mock_version.side_effect = [min_bq_version, pandas_version] + try: + gbq.to_gbq( + DataFrame(), 'dataset.tablename', project_id='my-project') + except gbq.TableCreationError: + pass + assert len(recwarn) == 0 + + def test_to_gbq_with_verbose_old_pandas_no_warnings(self, recwarn): + import pkg_resources + min_bq_version = pkg_resources.parse_version('0.29.0') + pandas_version = pkg_resources.parse_version('0.22.0') + with mock.patch( + 'pkg_resources.Distribution.parsed_version', + new_callable=mock.PropertyMock) as mock_version: + mock_version.side_effect = [min_bq_version, pandas_version] + try: + gbq.to_gbq( + DataFrame(), + 'dataset.tablename', + project_id='my-project', + verbose=True) + except gbq.TableCreationError: + pass + assert len(recwarn) == 0 + def test_read_gbq_with_no_project_id_given_should_fail(self): with pytest.raises(TypeError): gbq.read_gbq('SELECT 1') @@ -359,6 +460,50 @@ def test_read_gbq_with_corrupted_private_key_json_should_fail(self): 'SELECT 1', project_id='x', private_key=re.sub('[a-z]', '9', _get_private_key_contents())) + def test_read_gbq_with_verbose_new_pandas_warns_deprecation(self): + import pkg_resources + min_bq_version = pkg_resources.parse_version('0.29.0') + pandas_version = pkg_resources.parse_version('0.23.0') + with pytest.warns(FutureWarning), \ + mock.patch( + 'pkg_resources.Distribution.parsed_version', + new_callable=mock.PropertyMock) as mock_version: + mock_version.side_effect = [min_bq_version, pandas_version] + gbq.read_gbq('SELECT 1', project_id='my-project', verbose=True) + + def test_read_gbq_with_not_verbose_new_pandas_warns_deprecation(self): + import pkg_resources + min_bq_version = pkg_resources.parse_version('0.29.0') + pandas_version = pkg_resources.parse_version('0.23.0') + with pytest.warns(FutureWarning), \ + mock.patch( + 'pkg_resources.Distribution.parsed_version', + new_callable=mock.PropertyMock) as mock_version: + mock_version.side_effect = [min_bq_version, pandas_version] + gbq.read_gbq('SELECT 1', project_id='my-project', verbose=False) + + def test_read_gbq_wo_verbose_w_new_pandas_no_warnings(self, recwarn): + import pkg_resources + min_bq_version = pkg_resources.parse_version('0.29.0') + pandas_version = pkg_resources.parse_version('0.23.0') + with mock.patch( + 'pkg_resources.Distribution.parsed_version', + new_callable=mock.PropertyMock) as mock_version: + mock_version.side_effect = [min_bq_version, pandas_version] + gbq.read_gbq('SELECT 1', project_id='my-project') + assert len(recwarn) == 0 + + def test_read_gbq_with_verbose_old_pandas_no_warnings(self, recwarn): + import pkg_resources + min_bq_version = pkg_resources.parse_version('0.29.0') + pandas_version = pkg_resources.parse_version('0.22.0') + with mock.patch( + 'pkg_resources.Distribution.parsed_version', + new_callable=mock.PropertyMock) as mock_version: + mock_version.side_effect = [min_bq_version, pandas_version] + gbq.read_gbq('SELECT 1', project_id='my-project', verbose=True) + assert len(recwarn) == 0 + def test_should_read(project, credentials): From 2e7c0d933f53f297a25cf983a84dd18db93261df Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 6 Apr 2018 17:14:59 -0700 Subject: [PATCH 122/519] TST: use nox for testing (#160) * TST: use nox for testing Separates conda install from pip installs in Travis config. The reason is that nox isn't playing nice with conda. I got a "could not find _remove_dead_weakref" when trying to install pip packages when conda was also present. * TST: add global pyenv so binaries appear with right aliases. --- packages/pandas-gbq/.gitignore | 1 + packages/pandas-gbq/.travis.yml | 68 ++++++++-------- .../pandas-gbq/ci/requirements-3.5-0.18.1.pip | 1 - .../ci/requirements-3.6-0.20.1.conda | 1 - .../pandas-gbq/ci/requirements-3.6-MASTER.pip | 4 +- packages/pandas-gbq/docs/source/changelog.rst | 5 ++ .../pandas-gbq/docs/source/contributing.rst | 13 +++ packages/pandas-gbq/nox.py | 81 +++++++++++++++++++ .../pandas_gbq/tests/test__query.py | 5 +- packages/pandas-gbq/requirements-dev.txt | 6 ++ 10 files changed, 148 insertions(+), 37 deletions(-) create mode 100644 packages/pandas-gbq/nox.py create mode 100644 packages/pandas-gbq/requirements-dev.txt diff --git a/packages/pandas-gbq/.gitignore b/packages/pandas-gbq/.gitignore index 147e7e1e5bee..33e26ed40d6e 100644 --- a/packages/pandas-gbq/.gitignore +++ b/packages/pandas-gbq/.gitignore @@ -68,6 +68,7 @@ dist **/wheelhouse/* # coverage .coverage +.nox # OS generated files # ###################### diff --git a/packages/pandas-gbq/.travis.yml b/packages/pandas-gbq/.travis.yml index 423786805e6c..bce116d22bc2 100644 --- a/packages/pandas-gbq/.travis.yml +++ b/packages/pandas-gbq/.travis.yml @@ -3,48 +3,50 @@ sudo: false language: python env: - - PYTHON=2.7 PANDAS=0.19.2 COVERAGE='false' LINT='true' - - PYTHON=3.5 PANDAS=0.18.1 COVERAGE='true' LINT='false' - - PYTHON=3.6 PANDAS=0.20.1 COVERAGE='false' LINT='false' - - PYTHON=3.6 PANDAS=MASTER COVERAGE='false' LINT='true' + - PYTHON=2.7 PANDAS=0.19.2 COVERAGE='false' LINT='true' PYENV_VERSION=2.7.14 + - PYTHON=3.5 PANDAS=0.18.1 COVERAGE='true' LINT='false' PYENV_VERSION=3.5.4 + - PYTHON=3.6 PANDAS=0.20.1 COVERAGE='false' LINT='false' PYENV_VERSION=3.6.1 + - PYTHON=3.6 PANDAS=MASTER COVERAGE='false' LINT='true' PYENV_VERSION=3.6.1 before_install: - echo "before_install" - source ci/travis_process_gbq_encryption.sh install: - - wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; - - bash miniconda.sh -b -p $HOME/miniconda - - export PATH="$HOME/miniconda/bin:$PATH" - - hash -r - - conda config --set always_yes yes --set changeps1 no - - conda config --add channels pandas - - conda config --add channels conda-forge - - conda update -q conda - - conda info -a - - conda create -q -n test-environment python=$PYTHON - - source activate test-environment - - if [[ "$PANDAS" == "MASTER" ]]; then - conda install -q numpy pytz python-dateutil; - PRE_WHEELS="https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com"; - pip install --pre --upgrade --timeout=60 -f $PRE_WHEELS pandas; - pip install -e 'git+https://github.com/GoogleCloudPlatform/google-cloud-python.git#egg=version_subpkg&subdirectory=api_core'; - pip install -e 'git+https://github.com/GoogleCloudPlatform/google-cloud-python.git#egg=version_subpkg&subdirectory=core'; - pip install -e 'git+https://github.com/GoogleCloudPlatform/google-cloud-python.git#egg=version_subpkg&subdirectory=bigquery'; + # work around https://github.com/travis-ci/travis-ci/issues/8363 + # https://github.com/pre-commit/pre-commit/commit/e3ab8902692e896da9ded42bd4d76ea4e1de359d + - pyenv install -s $PYENV_VERSION + - pyenv global system $PYENV_VERSION + - REQ="ci/requirements-${PYTHON}-${PANDAS}" ; + if [ -f "$REQ.pip" ]; then + pip install --upgrade nox-automation ; else + wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; + bash miniconda.sh -b -p $HOME/miniconda ; + export PATH="$HOME/miniconda/bin:$PATH" ; + hash -r ; + conda config --set always_yes yes --set changeps1 no ; + conda config --add channels pandas ; + conda config --add channels conda-forge ; + conda update -q conda ; + conda info -a ; + conda create -q -n test-environment python=$PYTHON ; + source activate test-environment ; conda install -q pandas=$PANDAS; - fi - - pip install coverage pytest pytest-cov flake8 codecov - - REQ="ci/requirements-${PYTHON}-${PANDAS}" - - if [ -f "$REQ.pip" ]; then - pip install -r "$REQ.pip"; - else + pip install coverage pytest pytest-cov flake8 codecov ; conda install -q --file "$REQ.conda"; + conda list ; + python setup.py install ; fi - - conda list - - python setup.py install script: - - pytest -v --cov=pandas_gbq --cov-report xml:/tmp/pytest-cov.xml pandas_gbq - - if [[ $COVERAGE == 'true' ]]; then codecov ; fi - - if [[ $LINT == 'true' ]]; then flake8 pandas_gbq -v ; fi + - if [[ $PYTHON == '2.7' ]]; then nox -s test27 ; fi + - if [[ $PYTHON == '3.5' ]]; then nox -s test35 ; fi + - if [[ $PYTHON == '3.6' ]] && [[ "$PANDAS" == "MASTER" ]]; then nox -s test36master ; fi + - REQ="ci/requirements-${PYTHON}-${PANDAS}" ; + if [ -f "$REQ.conda" ]; then + pip install coverage pytest pytest-cov codecov ; + pytest -v --cov=pandas_gbq --cov-report xml:/tmp/pytest-cov.xml pandas_gbq ; + fi + - if [[ $COVERAGE == 'true' ]]; then nox -s cover ; fi + - if [[ $LINT == 'true' ]]; then nox -s lint ; fi diff --git a/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip b/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip index 68ac370dc0d4..dd33895c9e57 100644 --- a/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip +++ b/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip @@ -1,4 +1,3 @@ google-auth==1.4.1 google-auth-oauthlib==0.0.1 -mock google-cloud-bigquery==0.29.0 diff --git a/packages/pandas-gbq/ci/requirements-3.6-0.20.1.conda b/packages/pandas-gbq/ci/requirements-3.6-0.20.1.conda index b52f2aeb3289..e51ca487e760 100644 --- a/packages/pandas-gbq/ci/requirements-3.6-0.20.1.conda +++ b/packages/pandas-gbq/ci/requirements-3.6-0.20.1.conda @@ -1,4 +1,3 @@ google-auth google-auth-oauthlib -mock google-cloud-bigquery diff --git a/packages/pandas-gbq/ci/requirements-3.6-MASTER.pip b/packages/pandas-gbq/ci/requirements-3.6-MASTER.pip index 78f6834f550d..bd379b8bbdcb 100644 --- a/packages/pandas-gbq/ci/requirements-3.6-MASTER.pip +++ b/packages/pandas-gbq/ci/requirements-3.6-MASTER.pip @@ -1,3 +1,5 @@ google-auth google-auth-oauthlib -mock +git+https://github.com/GoogleCloudPlatform/google-cloud-python.git#egg=version_subpkg&subdirectory=api_core +git+https://github.com/GoogleCloudPlatform/google-cloud-python.git#egg=version_subpkg&subdirectory=core +git+https://github.com/GoogleCloudPlatform/google-cloud-python.git#egg=version_subpkg&subdirectory=bigquery diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index ffa62f1a7a4a..f771144210e0 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,6 +1,11 @@ Changelog ========= +0.5.0 / TBD +----------- + +- Tests now use `nox` to run in multiple Python environments. (:issue:`52`) + 0.4.1 / 2018-04-05 ------------------ diff --git a/packages/pandas-gbq/docs/source/contributing.rst b/packages/pandas-gbq/docs/source/contributing.rst index 986e61edff19..e6467eab81a5 100644 --- a/packages/pandas-gbq/docs/source/contributing.rst +++ b/packages/pandas-gbq/docs/source/contributing.rst @@ -268,6 +268,19 @@ Or with one of the following constructs:: For more, see the `pytest `_ documentation. +Testing on multiple Python versions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +pandas-gbq uses `nox `__ to automate testing in +multiple Python environments. First, install nox. + +.. code-block:: shell + + $ pip install --upgrade nox-automation + +To run tests in all versions of Python, run `nox` from the repository's root +directory. + .. _contributing.gbq_integration_tests: Running Google BigQuery Integration Tests diff --git a/packages/pandas-gbq/nox.py b/packages/pandas-gbq/nox.py new file mode 100644 index 000000000000..4de95fe9c291 --- /dev/null +++ b/packages/pandas-gbq/nox.py @@ -0,0 +1,81 @@ +"""Nox test automation configuration. + +See: https://nox.readthedocs.io/en/latest/ +""" + +import os.path + +import nox + + +PANDAS_PRE_WHEELS = ( + 'https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83' + '.ssl.cf2.rackcdn.com') + + +@nox.session +def default(session): + session.install('mock', 'pytest', 'pytest-cov') + session.install('-e', '.') + session.run( + 'pytest', + os.path.join('pandas_gbq', 'tests'), + '--quiet', + '--cov=pandas_gbq', + '--cov-report', + 'xml:/tmp/pytest-cov.xml', + *session.posargs + ) + + +@nox.session +def test27(session): + session.interpreter = 'python2.7' + session.install( + '-r', os.path.join('.', 'ci', 'requirements-2.7-0.19.2.pip')) + default(session) + + +@nox.session +def test35(session): + session.interpreter = 'python3.5' + session.install( + '-r', os.path.join('.', 'ci', 'requirements-3.5-0.18.1.pip')) + default(session) + + +@nox.session +def test36(session): + session.interpreter = 'python3.6' + session.install( + '-r', os.path.join('.', 'ci', 'requirements-3.6-0.20.1.conda')) + default(session) + + +@nox.session +def test36master(session): + session.interpreter = 'python3.6' + session.install( + '--pre', + '--upgrade', + '--timeout=60', + '-f', PANDAS_PRE_WHEELS, + 'pandas') + session.install( + '-r', os.path.join('.', 'ci', 'requirements-3.6-MASTER.pip')) + default(session) + + +@nox.session +def lint(session): + session.install('flake8') + session.run('flake8', 'pandas_gbq', '-v') + + +@nox.session +def cover(session): + session.interpreter = 'python3.5' + + session.install('coverage', 'pytest-cov') + session.run('coverage', 'report', '--show-missing', '--fail-under=40') + session.run('coverage', 'erase') diff --git a/packages/pandas-gbq/pandas_gbq/tests/test__query.py b/packages/pandas-gbq/pandas_gbq/tests/test__query.py index 43ab00f3171d..092606367219 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test__query.py +++ b/packages/pandas-gbq/pandas_gbq/tests/test__query.py @@ -1,7 +1,10 @@ import pkg_resources -import mock +try: + import mock +except ImportError: + from unittest import mock @mock.patch('google.cloud.bigquery.QueryJobConfig') diff --git a/packages/pandas-gbq/requirements-dev.txt b/packages/pandas-gbq/requirements-dev.txt new file mode 100644 index 000000000000..e0e72d7fa5a9 --- /dev/null +++ b/packages/pandas-gbq/requirements-dev.txt @@ -0,0 +1,6 @@ +flake8 +google-cloud-bigquery +nox-automation +pandas +pytest +setuptools From bdcfb9ca949c8b9f313ba0ce2ec956c1f6f1115d Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 10 Apr 2018 16:10:54 -0700 Subject: [PATCH 123/519] CLN: rename internal modules. move tests. (#163) * CLN: rename internal modules. move tests. Per GH#154 we don't need to use underscore to show submodules are private. To be 100% clear, I have added a disclaimer to the API reference as well. Also, I have split the system tests from the unit tests so that the (fast) unit tests can be more easily run separately from the (slow) system tests. I have followed the same directory structure as used in the google-cloud-bigquery library. * CLN: move _version.py back for versioneer * CLN: run tests from tests directory in conda test run --- packages/pandas-gbq/.travis.yml | 4 +- packages/pandas-gbq/docs/source/api.rst | 5 + packages/pandas-gbq/docs/source/changelog.rst | 1 + packages/pandas-gbq/nox.py | 22 +- packages/pandas-gbq/pandas_gbq/gbq.py | 12 +- .../pandas_gbq/{_load.py => load.py} | 4 +- .../pandas_gbq/{_query.py => query.py} | 0 .../pandas_gbq/{_schema.py => schema.py} | 0 .../{pandas_gbq => }/tests/__init__.py | 0 .../tests/test_gbq.py => tests/system.py} | 230 +--------------- packages/pandas-gbq/tests/unit/__init__.py | 0 packages/pandas-gbq/tests/unit/test_gbq.py | 252 ++++++++++++++++++ .../test__load.py => tests/unit/test_load.py} | 12 +- .../unit/test_query.py} | 16 +- .../unit/test_schema.py} | 4 +- 15 files changed, 302 insertions(+), 260 deletions(-) rename packages/pandas-gbq/pandas_gbq/{_load.py => load.py} (96%) rename packages/pandas-gbq/pandas_gbq/{_query.py => query.py} (100%) rename packages/pandas-gbq/pandas_gbq/{_schema.py => schema.py} (100%) rename packages/pandas-gbq/{pandas_gbq => }/tests/__init__.py (100%) rename packages/pandas-gbq/{pandas_gbq/tests/test_gbq.py => tests/system.py} (85%) create mode 100644 packages/pandas-gbq/tests/unit/__init__.py create mode 100644 packages/pandas-gbq/tests/unit/test_gbq.py rename packages/pandas-gbq/{pandas_gbq/tests/test__load.py => tests/unit/test_load.py} (78%) rename packages/pandas-gbq/{pandas_gbq/tests/test__query.py => tests/unit/test_query.py} (83%) rename packages/pandas-gbq/{pandas_gbq/tests/test__schema.py => tests/unit/test_schema.py} (94%) diff --git a/packages/pandas-gbq/.travis.yml b/packages/pandas-gbq/.travis.yml index bce116d22bc2..91287092c7b0 100644 --- a/packages/pandas-gbq/.travis.yml +++ b/packages/pandas-gbq/.travis.yml @@ -45,8 +45,8 @@ script: - if [[ $PYTHON == '3.6' ]] && [[ "$PANDAS" == "MASTER" ]]; then nox -s test36master ; fi - REQ="ci/requirements-${PYTHON}-${PANDAS}" ; if [ -f "$REQ.conda" ]; then - pip install coverage pytest pytest-cov codecov ; - pytest -v --cov=pandas_gbq --cov-report xml:/tmp/pytest-cov.xml pandas_gbq ; + pip install pytest ; + pytest -v tests ; fi - if [[ $COVERAGE == 'true' ]]; then nox -s cover ; fi - if [[ $LINT == 'true' ]]; then nox -s lint ; fi diff --git a/packages/pandas-gbq/docs/source/api.rst b/packages/pandas-gbq/docs/source/api.rst index d1f50b9a8ac4..f5bcf9576bce 100644 --- a/packages/pandas-gbq/docs/source/api.rst +++ b/packages/pandas-gbq/docs/source/api.rst @@ -5,6 +5,11 @@ API Reference ************* +.. note:: + + Only functions and classes which are members of the ``pandas_gbq`` module + are considered public. Submodules and their members are considered private. + .. autosummary:: read_gbq diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index f771144210e0..a454ec84a300 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -5,6 +5,7 @@ Changelog ----------- - Tests now use `nox` to run in multiple Python environments. (:issue:`52`) +- Renamed internal modules. (:issue:`154`) 0.4.1 / 2018-04-05 ------------------ diff --git a/packages/pandas-gbq/nox.py b/packages/pandas-gbq/nox.py index 4de95fe9c291..7718b75c2029 100644 --- a/packages/pandas-gbq/nox.py +++ b/packages/pandas-gbq/nox.py @@ -19,9 +19,27 @@ def default(session): session.install('-e', '.') session.run( 'pytest', - os.path.join('pandas_gbq', 'tests'), + os.path.join('.', 'tests', 'unit'), + os.path.join('.', 'tests', 'system.py'), '--quiet', '--cov=pandas_gbq', + '--cov=tests.unit', + '--cov-report', + 'xml:/tmp/pytest-cov.xml', + *session.posargs + ) + + +@nox.session +def unit(session): + session.install('mock', 'pytest', 'pytest-cov') + session.install('-e', '.') + session.run( + 'pytest', + os.path.join('.', 'tests', 'unit'), + '--quiet', + '--cov=pandas_gbq', + '--cov=tests.unit', '--cov-report', 'xml:/tmp/pytest-cov.xml', *session.posargs @@ -69,7 +87,7 @@ def test36master(session): @nox.session def lint(session): session.install('flake8') - session.run('flake8', 'pandas_gbq', '-v') + session.run('flake8', 'pandas_gbq', 'tests', '-v') @nox.session diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index f8bbaf22d7ec..5c8af053abe9 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -458,7 +458,7 @@ def process_http_error(ex): def run_query(self, query, **kwargs): from google.auth.exceptions import RefreshError from concurrent.futures import TimeoutError - from pandas_gbq import _query + import pandas_gbq.query job_config = { 'query': { @@ -484,7 +484,7 @@ def run_query(self, query, **kwargs): logger.info('Requesting query... ') query_reply = self.client.query( query, - job_config=_query.query_config( + job_config=pandas_gbq.query.query_config( job_config, BIGQUERY_INSTALLED_VERSION)) logger.info('ok.\nQuery running...') except (RefreshError, ValueError): @@ -552,13 +552,13 @@ def run_query(self, query, **kwargs): def load_data( self, dataframe, dataset_id, table_id, chunksize=None, schema=None): - from pandas_gbq import _load + from pandas_gbq import load total_rows = len(dataframe) logger.info("\n\n") try: - for remaining_rows in _load.load_chunks( + for remaining_rows in load.load_chunks( self.client, dataframe, dataset_id, table_id, chunksize=chunksize, schema=schema): logger.info("\rLoad is {0}% Complete".format( @@ -1000,8 +1000,8 @@ def generate_bq_schema(df, default_type='STRING'): def _generate_bq_schema(df, default_type='STRING'): - from pandas_gbq import _schema - return _schema.generate_bq_schema(df, default_type=default_type) + from pandas_gbq import schema + return schema.generate_bq_schema(df, default_type=default_type) class _Table(GbqConnector): diff --git a/packages/pandas-gbq/pandas_gbq/_load.py b/packages/pandas-gbq/pandas_gbq/load.py similarity index 96% rename from packages/pandas-gbq/pandas_gbq/_load.py rename to packages/pandas-gbq/pandas_gbq/load.py index c04bf7a6a8d6..2adb59f6edf6 100644 --- a/packages/pandas-gbq/pandas_gbq/_load.py +++ b/packages/pandas-gbq/pandas_gbq/load.py @@ -3,7 +3,7 @@ import six from google.cloud import bigquery -from pandas_gbq import _schema +import pandas_gbq.schema def encode_chunk(dataframe): @@ -51,7 +51,7 @@ def load_chunks( job_config.source_format = 'CSV' if schema is None: - schema = _schema.generate_bq_schema(dataframe) + schema = pandas_gbq.schema.generate_bq_schema(dataframe) # Manually create the schema objects, adding NULLABLE mode # as a workaround for diff --git a/packages/pandas-gbq/pandas_gbq/_query.py b/packages/pandas-gbq/pandas_gbq/query.py similarity index 100% rename from packages/pandas-gbq/pandas_gbq/_query.py rename to packages/pandas-gbq/pandas_gbq/query.py diff --git a/packages/pandas-gbq/pandas_gbq/_schema.py b/packages/pandas-gbq/pandas_gbq/schema.py similarity index 100% rename from packages/pandas-gbq/pandas_gbq/_schema.py rename to packages/pandas-gbq/pandas_gbq/schema.py diff --git a/packages/pandas-gbq/pandas_gbq/tests/__init__.py b/packages/pandas-gbq/tests/__init__.py similarity index 100% rename from packages/pandas-gbq/pandas_gbq/tests/__init__.py rename to packages/pandas-gbq/tests/__init__.py diff --git a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py b/packages/pandas-gbq/tests/system.py similarity index 85% rename from packages/pandas-gbq/pandas_gbq/tests/test_gbq.py rename to packages/pandas-gbq/tests/system.py index 93c48b0c1ac9..8782b5801f98 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test_gbq.py +++ b/packages/pandas-gbq/tests/system.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import os -import re import sys from datetime import datetime from random import randint @@ -13,13 +12,12 @@ import pytz from pandas import DataFrame, NaT, compat from pandas.compat import range, u -from pandas.compat.numpy import np_datetime64_compat from pandas_gbq import gbq try: import mock -except ImportError: +except ImportError: # pragma: NO COVER from unittest import mock TABLE_ID = 'new_test' @@ -280,233 +278,7 @@ def test_get_user_account_credentials_returns_credentials(self): assert isinstance(credentials, Credentials) -class TestGBQUnit(object): - - @pytest.fixture(autouse=True) - def mock_bigquery_client(self, monkeypatch): - import google.cloud.bigquery - import google.cloud.bigquery.table - mock_client = mock.create_autospec(google.cloud.bigquery.Client) - # Mock out SELECT 1 query results. - mock_query = mock.create_autospec(google.cloud.bigquery.QueryJob) - mock_query.state = 'DONE' - mock_rows = mock.create_autospec( - google.cloud.bigquery.table.RowIterator) - mock_rows.total_rows = 1 - mock_rows.schema = [ - google.cloud.bigquery.SchemaField('_f0', 'INTEGER')] - mock_rows.__iter__.return_value = [(1,)] - mock_query.result.return_value = mock_rows - mock_client.query.return_value = mock_query - monkeypatch.setattr( - gbq.GbqConnector, 'get_client', lambda _: mock_client) - - @pytest.fixture(autouse=True) - def no_auth(self, monkeypatch): - import google.auth.credentials - mock_credentials = mock.create_autospec( - google.auth.credentials.Credentials) - monkeypatch.setattr( - gbq.GbqConnector, - 'get_application_default_credentials', - lambda _: mock_credentials) - monkeypatch.setattr( - gbq.GbqConnector, - 'get_user_account_credentials', - lambda _: mock_credentials) - - def test_should_return_credentials_path_set_by_env_var(self): - env = {'PANDAS_GBQ_CREDENTIALS_FILE': '/tmp/dummy.dat'} - with mock.patch.dict('os.environ', env): - assert gbq._get_credentials_file() == '/tmp/dummy.dat' - - @pytest.mark.parametrize( - ('input', 'type_', 'expected'), [ - (1, 'INTEGER', int(1)), - (1, 'FLOAT', float(1)), - pytest.param('false', 'BOOLEAN', False, marks=pytest.mark.xfail), - pytest.param( - '0e9', 'TIMESTAMP', - np_datetime64_compat('1970-01-01T00:00:00Z'), - marks=pytest.mark.xfail), - ('STRING', 'STRING', 'STRING'), - ]) - def test_should_return_bigquery_correctly_typed( - self, input, type_, expected): - result = gbq._parse_data( - dict(fields=[dict(name='x', type=type_, mode='NULLABLE')]), - rows=[[input]]).iloc[0, 0] - assert result == expected - - def test_to_gbq_should_fail_if_invalid_table_name_passed(self): - with pytest.raises(gbq.NotFoundException): - gbq.to_gbq(DataFrame(), 'invalid_table_name', project_id="1234") - - def test_to_gbq_with_no_project_id_given_should_fail(self): - with pytest.raises(TypeError): - gbq.to_gbq(DataFrame(), 'dataset.tablename') - - def test_to_gbq_with_verbose_new_pandas_warns_deprecation(self): - import pkg_resources - min_bq_version = pkg_resources.parse_version('0.29.0') - pandas_version = pkg_resources.parse_version('0.23.0') - with pytest.warns(FutureWarning), \ - mock.patch( - 'pkg_resources.Distribution.parsed_version', - new_callable=mock.PropertyMock) as mock_version: - mock_version.side_effect = [min_bq_version, pandas_version] - try: - gbq.to_gbq( - DataFrame(), - 'dataset.tablename', - project_id='my-project', - verbose=True) - except gbq.TableCreationError: - pass - - def test_to_gbq_with_not_verbose_new_pandas_warns_deprecation(self): - import pkg_resources - min_bq_version = pkg_resources.parse_version('0.29.0') - pandas_version = pkg_resources.parse_version('0.23.0') - with pytest.warns(FutureWarning), \ - mock.patch( - 'pkg_resources.Distribution.parsed_version', - new_callable=mock.PropertyMock) as mock_version: - mock_version.side_effect = [min_bq_version, pandas_version] - try: - gbq.to_gbq( - DataFrame(), - 'dataset.tablename', - project_id='my-project', - verbose=False) - except gbq.TableCreationError: - pass - - def test_to_gbq_wo_verbose_w_new_pandas_no_warnings(self, recwarn): - import pkg_resources - min_bq_version = pkg_resources.parse_version('0.29.0') - pandas_version = pkg_resources.parse_version('0.23.0') - with mock.patch( - 'pkg_resources.Distribution.parsed_version', - new_callable=mock.PropertyMock) as mock_version: - mock_version.side_effect = [min_bq_version, pandas_version] - try: - gbq.to_gbq( - DataFrame(), 'dataset.tablename', project_id='my-project') - except gbq.TableCreationError: - pass - assert len(recwarn) == 0 - - def test_to_gbq_with_verbose_old_pandas_no_warnings(self, recwarn): - import pkg_resources - min_bq_version = pkg_resources.parse_version('0.29.0') - pandas_version = pkg_resources.parse_version('0.22.0') - with mock.patch( - 'pkg_resources.Distribution.parsed_version', - new_callable=mock.PropertyMock) as mock_version: - mock_version.side_effect = [min_bq_version, pandas_version] - try: - gbq.to_gbq( - DataFrame(), - 'dataset.tablename', - project_id='my-project', - verbose=True) - except gbq.TableCreationError: - pass - assert len(recwarn) == 0 - - def test_read_gbq_with_no_project_id_given_should_fail(self): - with pytest.raises(TypeError): - gbq.read_gbq('SELECT 1') - - def test_that_parse_data_works_properly(self): - - from google.cloud.bigquery.table import Row - test_schema = {'fields': [ - {'mode': 'NULLABLE', 'name': 'column_x', 'type': 'STRING'}]} - field_to_index = {'column_x': 0} - values = ('row_value',) - test_page = [Row(values, field_to_index)] - - test_output = gbq._parse_data(test_schema, test_page) - correct_output = DataFrame({'column_x': ['row_value']}) - tm.assert_frame_equal(test_output, correct_output) - - def test_read_gbq_with_invalid_private_key_json_should_fail(self): - with pytest.raises(gbq.InvalidPrivateKeyFormat): - gbq.read_gbq('SELECT 1', project_id='x', private_key='y') - - def test_read_gbq_with_empty_private_key_json_should_fail(self): - with pytest.raises(gbq.InvalidPrivateKeyFormat): - gbq.read_gbq('SELECT 1', project_id='x', private_key='{}') - - def test_read_gbq_with_private_key_json_wrong_types_should_fail(self): - with pytest.raises(gbq.InvalidPrivateKeyFormat): - gbq.read_gbq( - 'SELECT 1', project_id='x', - private_key='{ "client_email" : 1, "private_key" : True }') - - def test_read_gbq_with_empty_private_key_file_should_fail(self): - with tm.ensure_clean() as empty_file_path: - with pytest.raises(gbq.InvalidPrivateKeyFormat): - gbq.read_gbq('SELECT 1', project_id='x', - private_key=empty_file_path) - - def test_read_gbq_with_corrupted_private_key_json_should_fail(self): - _skip_if_no_private_key_contents() - - with pytest.raises(gbq.InvalidPrivateKeyFormat): - gbq.read_gbq( - 'SELECT 1', project_id='x', - private_key=re.sub('[a-z]', '9', _get_private_key_contents())) - - def test_read_gbq_with_verbose_new_pandas_warns_deprecation(self): - import pkg_resources - min_bq_version = pkg_resources.parse_version('0.29.0') - pandas_version = pkg_resources.parse_version('0.23.0') - with pytest.warns(FutureWarning), \ - mock.patch( - 'pkg_resources.Distribution.parsed_version', - new_callable=mock.PropertyMock) as mock_version: - mock_version.side_effect = [min_bq_version, pandas_version] - gbq.read_gbq('SELECT 1', project_id='my-project', verbose=True) - - def test_read_gbq_with_not_verbose_new_pandas_warns_deprecation(self): - import pkg_resources - min_bq_version = pkg_resources.parse_version('0.29.0') - pandas_version = pkg_resources.parse_version('0.23.0') - with pytest.warns(FutureWarning), \ - mock.patch( - 'pkg_resources.Distribution.parsed_version', - new_callable=mock.PropertyMock) as mock_version: - mock_version.side_effect = [min_bq_version, pandas_version] - gbq.read_gbq('SELECT 1', project_id='my-project', verbose=False) - - def test_read_gbq_wo_verbose_w_new_pandas_no_warnings(self, recwarn): - import pkg_resources - min_bq_version = pkg_resources.parse_version('0.29.0') - pandas_version = pkg_resources.parse_version('0.23.0') - with mock.patch( - 'pkg_resources.Distribution.parsed_version', - new_callable=mock.PropertyMock) as mock_version: - mock_version.side_effect = [min_bq_version, pandas_version] - gbq.read_gbq('SELECT 1', project_id='my-project') - assert len(recwarn) == 0 - - def test_read_gbq_with_verbose_old_pandas_no_warnings(self, recwarn): - import pkg_resources - min_bq_version = pkg_resources.parse_version('0.29.0') - pandas_version = pkg_resources.parse_version('0.22.0') - with mock.patch( - 'pkg_resources.Distribution.parsed_version', - new_callable=mock.PropertyMock) as mock_version: - mock_version.side_effect = [min_bq_version, pandas_version] - gbq.read_gbq('SELECT 1', project_id='my-project', verbose=True) - assert len(recwarn) == 0 - - def test_should_read(project, credentials): - query = 'SELECT "PI" AS valid_string' df = gbq.read_gbq(query, project_id=project, private_key=credentials) tm.assert_frame_equal(df, DataFrame({'valid_string': ['PI']})) diff --git a/packages/pandas-gbq/tests/unit/__init__.py b/packages/pandas-gbq/tests/unit/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py new file mode 100644 index 000000000000..b7febafd9873 --- /dev/null +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -0,0 +1,252 @@ +# -*- coding: utf-8 -*- + +import pandas.util.testing as tm +import pytest +from pandas import DataFrame +from pandas.compat.numpy import np_datetime64_compat + +from pandas_gbq import gbq + +try: + import mock +except ImportError: # pragma: NO COVER + from unittest import mock + + +@pytest.fixture(autouse=True) +def mock_bigquery_client(monkeypatch): + import google.cloud.bigquery + import google.cloud.bigquery.table + mock_client = mock.create_autospec(google.cloud.bigquery.Client) + # Mock out SELECT 1 query results. + mock_query = mock.create_autospec(google.cloud.bigquery.QueryJob) + mock_query.state = 'DONE' + mock_rows = mock.create_autospec( + google.cloud.bigquery.table.RowIterator) + mock_rows.total_rows = 1 + mock_rows.schema = [ + google.cloud.bigquery.SchemaField('_f0', 'INTEGER')] + mock_rows.__iter__.return_value = [(1,)] + mock_query.result.return_value = mock_rows + mock_client.query.return_value = mock_query + monkeypatch.setattr( + gbq.GbqConnector, 'get_client', lambda _: mock_client) + + +@pytest.fixture(autouse=True) +def no_auth(monkeypatch): + import google.auth.credentials + mock_credentials = mock.create_autospec( + google.auth.credentials.Credentials) + monkeypatch.setattr( + gbq.GbqConnector, + 'get_application_default_credentials', + lambda _: mock_credentials) + monkeypatch.setattr( + gbq.GbqConnector, + 'get_user_account_credentials', + lambda _: mock_credentials) + + +def test_should_return_credentials_path_set_by_env_var(): + env = {'PANDAS_GBQ_CREDENTIALS_FILE': '/tmp/dummy.dat'} + with mock.patch.dict('os.environ', env): + assert gbq._get_credentials_file() == '/tmp/dummy.dat' + + +@pytest.mark.parametrize( + ('input', 'type_', 'expected'), [ + (1, 'INTEGER', int(1)), + (1, 'FLOAT', float(1)), + pytest.param('false', 'BOOLEAN', False, marks=pytest.mark.xfail), + pytest.param( + '0e9', 'TIMESTAMP', + np_datetime64_compat('1970-01-01T00:00:00Z'), + marks=pytest.mark.xfail), + ('STRING', 'STRING', 'STRING'), + ]) +def test_should_return_bigquery_correctly_typed( + input, type_, expected): + result = gbq._parse_data( + dict(fields=[dict(name='x', type=type_, mode='NULLABLE')]), + rows=[[input]]).iloc[0, 0] + assert result == expected + + +def test_to_gbq_should_fail_if_invalid_table_name_passed(): + with pytest.raises(gbq.NotFoundException): + gbq.to_gbq(DataFrame(), 'invalid_table_name', project_id="1234") + + +def test_to_gbq_with_no_project_id_given_should_fail(): + with pytest.raises(TypeError): + gbq.to_gbq(DataFrame(), 'dataset.tablename') + + +def test_to_gbq_with_verbose_new_pandas_warns_deprecation(): + import pkg_resources + min_bq_version = pkg_resources.parse_version('0.29.0') + pandas_version = pkg_resources.parse_version('0.23.0') + with pytest.warns(FutureWarning), \ + mock.patch( + 'pkg_resources.Distribution.parsed_version', + new_callable=mock.PropertyMock) as mock_version: + mock_version.side_effect = [min_bq_version, pandas_version] + try: + gbq.to_gbq( + DataFrame(), + 'dataset.tablename', + project_id='my-project', + verbose=True) + except gbq.TableCreationError: + pass + + +def test_to_gbq_with_not_verbose_new_pandas_warns_deprecation(): + import pkg_resources + min_bq_version = pkg_resources.parse_version('0.29.0') + pandas_version = pkg_resources.parse_version('0.23.0') + with pytest.warns(FutureWarning), \ + mock.patch( + 'pkg_resources.Distribution.parsed_version', + new_callable=mock.PropertyMock) as mock_version: + mock_version.side_effect = [min_bq_version, pandas_version] + try: + gbq.to_gbq( + DataFrame(), + 'dataset.tablename', + project_id='my-project', + verbose=False) + except gbq.TableCreationError: + pass + + +def test_to_gbq_wo_verbose_w_new_pandas_no_warnings(recwarn): + import pkg_resources + min_bq_version = pkg_resources.parse_version('0.29.0') + pandas_version = pkg_resources.parse_version('0.23.0') + with mock.patch( + 'pkg_resources.Distribution.parsed_version', + new_callable=mock.PropertyMock) as mock_version: + mock_version.side_effect = [min_bq_version, pandas_version] + try: + gbq.to_gbq( + DataFrame(), 'dataset.tablename', project_id='my-project') + except gbq.TableCreationError: + pass + assert len(recwarn) == 0 + + +def test_to_gbq_with_verbose_old_pandas_no_warnings(recwarn): + import pkg_resources + min_bq_version = pkg_resources.parse_version('0.29.0') + pandas_version = pkg_resources.parse_version('0.22.0') + with mock.patch( + 'pkg_resources.Distribution.parsed_version', + new_callable=mock.PropertyMock) as mock_version: + mock_version.side_effect = [min_bq_version, pandas_version] + try: + gbq.to_gbq( + DataFrame(), + 'dataset.tablename', + project_id='my-project', + verbose=True) + except gbq.TableCreationError: + pass + assert len(recwarn) == 0 + + +def test_read_gbq_with_no_project_id_given_should_fail(): + with pytest.raises(TypeError): + gbq.read_gbq('SELECT 1') + + +def test_that_parse_data_works_properly(): + from google.cloud.bigquery.table import Row + test_schema = {'fields': [ + {'mode': 'NULLABLE', 'name': 'column_x', 'type': 'STRING'}]} + field_to_index = {'column_x': 0} + values = ('row_value',) + test_page = [Row(values, field_to_index)] + + test_output = gbq._parse_data(test_schema, test_page) + correct_output = DataFrame({'column_x': ['row_value']}) + tm.assert_frame_equal(test_output, correct_output) + + +def test_read_gbq_with_invalid_private_key_json_should_fail(): + with pytest.raises(gbq.InvalidPrivateKeyFormat): + gbq.read_gbq('SELECT 1', project_id='x', private_key='y') + + +def test_read_gbq_with_empty_private_key_json_should_fail(): + with pytest.raises(gbq.InvalidPrivateKeyFormat): + gbq.read_gbq('SELECT 1', project_id='x', private_key='{}') + + +def test_read_gbq_with_private_key_json_wrong_types_should_fail(): + with pytest.raises(gbq.InvalidPrivateKeyFormat): + gbq.read_gbq( + 'SELECT 1', project_id='x', + private_key='{ "client_email" : 1, "private_key" : True }') + + +def test_read_gbq_with_empty_private_key_file_should_fail(): + with tm.ensure_clean() as empty_file_path: + with pytest.raises(gbq.InvalidPrivateKeyFormat): + gbq.read_gbq('SELECT 1', project_id='x', + private_key=empty_file_path) + + +def test_read_gbq_with_corrupted_private_key_json_should_fail(): + with pytest.raises(gbq.InvalidPrivateKeyFormat): + gbq.read_gbq( + 'SELECT 1', project_id='x', private_key='99999999999999999') + + +def test_read_gbq_with_verbose_new_pandas_warns_deprecation(): + import pkg_resources + min_bq_version = pkg_resources.parse_version('0.29.0') + pandas_version = pkg_resources.parse_version('0.23.0') + with pytest.warns(FutureWarning), \ + mock.patch( + 'pkg_resources.Distribution.parsed_version', + new_callable=mock.PropertyMock) as mock_version: + mock_version.side_effect = [min_bq_version, pandas_version] + gbq.read_gbq('SELECT 1', project_id='my-project', verbose=True) + + +def test_read_gbq_with_not_verbose_new_pandas_warns_deprecation(): + import pkg_resources + min_bq_version = pkg_resources.parse_version('0.29.0') + pandas_version = pkg_resources.parse_version('0.23.0') + with pytest.warns(FutureWarning), \ + mock.patch( + 'pkg_resources.Distribution.parsed_version', + new_callable=mock.PropertyMock) as mock_version: + mock_version.side_effect = [min_bq_version, pandas_version] + gbq.read_gbq('SELECT 1', project_id='my-project', verbose=False) + + +def test_read_gbq_wo_verbose_w_new_pandas_no_warnings(recwarn): + import pkg_resources + min_bq_version = pkg_resources.parse_version('0.29.0') + pandas_version = pkg_resources.parse_version('0.23.0') + with mock.patch( + 'pkg_resources.Distribution.parsed_version', + new_callable=mock.PropertyMock) as mock_version: + mock_version.side_effect = [min_bq_version, pandas_version] + gbq.read_gbq('SELECT 1', project_id='my-project') + assert len(recwarn) == 0 + + +def test_read_gbq_with_verbose_old_pandas_no_warnings(recwarn): + import pkg_resources + min_bq_version = pkg_resources.parse_version('0.29.0') + pandas_version = pkg_resources.parse_version('0.22.0') + with mock.patch( + 'pkg_resources.Distribution.parsed_version', + new_callable=mock.PropertyMock) as mock_version: + mock_version.side_effect = [min_bq_version, pandas_version] + gbq.read_gbq('SELECT 1', project_id='my-project', verbose=True) + assert len(recwarn) == 0 diff --git a/packages/pandas-gbq/pandas_gbq/tests/test__load.py b/packages/pandas-gbq/tests/unit/test_load.py similarity index 78% rename from packages/pandas-gbq/pandas_gbq/tests/test__load.py rename to packages/pandas-gbq/tests/unit/test_load.py index d63638e2fa39..398a01ae23f2 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test__load.py +++ b/packages/pandas-gbq/tests/unit/test_load.py @@ -3,27 +3,26 @@ import numpy import pandas +from pandas_gbq import load + def test_encode_chunk_with_unicode(): """Test that a dataframe containing unicode can be encoded as a file. See: https://github.com/pydata/pandas-gbq/issues/106 """ - from pandas_gbq._load import encode_chunk - df = pandas.DataFrame( numpy.random.randn(6, 4), index=range(6), columns=list('ABCD')) df['s'] = u'信用卡' - csv_buffer = encode_chunk(df) + csv_buffer = load.encode_chunk(df) csv_bytes = csv_buffer.read() csv_string = csv_bytes.decode('utf-8') assert u'信用卡' in csv_string def test_encode_chunks_splits_dataframe(): - from pandas_gbq._load import encode_chunks df = pandas.DataFrame(numpy.random.randn(6, 4), index=range(6)) - chunks = list(encode_chunks(df, chunksize=2)) + chunks = list(load.encode_chunks(df, chunksize=2)) assert len(chunks) == 3 remaining, buffer = chunks[0] assert remaining == 4 @@ -31,9 +30,8 @@ def test_encode_chunks_splits_dataframe(): def test_encode_chunks_with_chunksize_none(): - from pandas_gbq._load import encode_chunks df = pandas.DataFrame(numpy.random.randn(6, 4), index=range(6)) - chunks = list(encode_chunks(df)) + chunks = list(load.encode_chunks(df)) assert len(chunks) == 1 remaining, buffer = chunks[0] assert remaining == 0 diff --git a/packages/pandas-gbq/pandas_gbq/tests/test__query.py b/packages/pandas-gbq/tests/unit/test_query.py similarity index 83% rename from packages/pandas-gbq/pandas_gbq/tests/test__query.py rename to packages/pandas-gbq/tests/unit/test_query.py index 092606367219..0a89dfb92545 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test__query.py +++ b/packages/pandas-gbq/tests/unit/test_query.py @@ -3,25 +3,23 @@ try: import mock -except ImportError: +except ImportError: # pragma: NO COVER from unittest import mock +from pandas_gbq import query + @mock.patch('google.cloud.bigquery.QueryJobConfig') def test_query_config_w_old_bq_version(mock_config): - from pandas_gbq._query import query_config - old_version = pkg_resources.parse_version('0.29.0') - query_config({'query': {'useLegacySql': False}}, old_version) + query.query_config({'query': {'useLegacySql': False}}, old_version) mock_config.from_api_repr.assert_called_once_with({'useLegacySql': False}) @mock.patch('google.cloud.bigquery.QueryJobConfig') def test_query_config_w_dev_bq_version(mock_config): - from pandas_gbq._query import query_config - dev_version = pkg_resources.parse_version('0.32.0.dev1') - query_config( + query.query_config( { 'query': { 'useLegacySql': False, @@ -40,10 +38,8 @@ def test_query_config_w_dev_bq_version(mock_config): @mock.patch('google.cloud.bigquery.QueryJobConfig') def test_query_config_w_new_bq_version(mock_config): - from pandas_gbq._query import query_config - dev_version = pkg_resources.parse_version('1.0.0') - query_config( + query.query_config( { 'query': { 'useLegacySql': False, diff --git a/packages/pandas-gbq/pandas_gbq/tests/test__schema.py b/packages/pandas-gbq/tests/unit/test_schema.py similarity index 94% rename from packages/pandas-gbq/pandas_gbq/tests/test__schema.py rename to packages/pandas-gbq/tests/unit/test_schema.py index 5c7fffc1cbff..4d6b0add3958 100644 --- a/packages/pandas-gbq/pandas_gbq/tests/test__schema.py +++ b/packages/pandas-gbq/tests/unit/test_schema.py @@ -4,7 +4,7 @@ import pandas import pytest -from pandas_gbq import _schema +import pandas_gbq.schema @pytest.mark.parametrize( @@ -51,5 +51,5 @@ ), ]) def test_generate_bq_schema(dataframe, expected_schema): - schema = _schema.generate_bq_schema(dataframe) + schema = pandas_gbq.schema.generate_bq_schema(dataframe) assert schema == expected_schema From 2cd56037c81b0abec3eb3a5118413a4f97f91b5e Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 10 Apr 2018 16:45:15 -0700 Subject: [PATCH 124/519] CLN: update test paths in contributing guide (#164) --- packages/pandas-gbq/docs/source/contributing.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/pandas-gbq/docs/source/contributing.rst b/packages/pandas-gbq/docs/source/contributing.rst index e6467eab81a5..351fccd09962 100644 --- a/packages/pandas-gbq/docs/source/contributing.rst +++ b/packages/pandas-gbq/docs/source/contributing.rst @@ -250,7 +250,8 @@ Running the test suite The tests can then be run directly inside your Git clone (without having to install *pandas-gbq*) by typing:: - pytest pandas_gbq + pytest tests/unit + pytest tests/system.py The tests suite is exhaustive and takes around 20 minutes to run. Often it is worth running only a subset of tests first around your changes before running the @@ -258,13 +259,13 @@ entire suite. The easiest way to do this is with:: - pytest pandas_gbq/path/to/test.py -k regex_matching_test_name + pytest tests/path/to/test.py -k regex_matching_test_name Or with one of the following constructs:: - pytest pandas_gbq/tests/[test-module].py - pytest pandas_gbq/tests/[test-module].py::[TestClass] - pytest pandas_gbq/tests/[test-module].py::[TestClass]::[test_method] + pytest tests/[test-module].py + pytest tests/[test-module].py::[TestClass] + pytest tests/[test-module].py::[TestClass]::[test_method] For more, see the `pytest `_ documentation. From 4da9c97a264e87288b63868bf0ca12a6a4bd2200 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 24 Apr 2018 20:48:07 -0700 Subject: [PATCH 125/519] TST: mock job_id for g-c-bq 0.29 tests. pin g-c-bq version for conda build (#169) * TST: add mocked job_id for google-cloud-bigquery 0.29 Closes GH#168. * TST: upgrade setuptools for namespace package support * TST: upgrade pip for namespace package breakage. * TST: remove pip commands from conda test environment * TST: remove another pip command from conda tests * TST: add setuptools to conda installation * TST: add explicit google-api-core dep to conda build * TST: pin miniconda to specific version * TST: pin google-cloud-bigquery in conda build --- packages/pandas-gbq/.travis.yml | 9 ++++++--- packages/pandas-gbq/ci/requirements-3.6-0.20.1.conda | 7 ++++++- packages/pandas-gbq/tests/unit/test_gbq.py | 1 + 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/packages/pandas-gbq/.travis.yml b/packages/pandas-gbq/.travis.yml index 91287092c7b0..b30f79aa8e69 100644 --- a/packages/pandas-gbq/.travis.yml +++ b/packages/pandas-gbq/.travis.yml @@ -17,11 +17,15 @@ install: # https://github.com/pre-commit/pre-commit/commit/e3ab8902692e896da9ded42bd4d76ea4e1de359d - pyenv install -s $PYENV_VERSION - pyenv global system $PYENV_VERSION + # Upgrade setuptools and pip to work around + # https://github.com/pypa/setuptools/issues/885 + - pip install --upgrade setuptools + - pip install --upgrade pip - REQ="ci/requirements-${PYTHON}-${PANDAS}" ; if [ -f "$REQ.pip" ]; then pip install --upgrade nox-automation ; else - wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; + wget http://repo.continuum.io/miniconda/Miniconda3-4.3.30-Linux-x86_64.sh -O miniconda.sh; bash miniconda.sh -b -p $HOME/miniconda ; export PATH="$HOME/miniconda/bin:$PATH" ; hash -r ; @@ -32,8 +36,8 @@ install: conda info -a ; conda create -q -n test-environment python=$PYTHON ; source activate test-environment ; + conda install -q setuptools ; conda install -q pandas=$PANDAS; - pip install coverage pytest pytest-cov flake8 codecov ; conda install -q --file "$REQ.conda"; conda list ; python setup.py install ; @@ -45,7 +49,6 @@ script: - if [[ $PYTHON == '3.6' ]] && [[ "$PANDAS" == "MASTER" ]]; then nox -s test36master ; fi - REQ="ci/requirements-${PYTHON}-${PANDAS}" ; if [ -f "$REQ.conda" ]; then - pip install pytest ; pytest -v tests ; fi - if [[ $COVERAGE == 'true' ]]; then nox -s cover ; fi diff --git a/packages/pandas-gbq/ci/requirements-3.6-0.20.1.conda b/packages/pandas-gbq/ci/requirements-3.6-0.20.1.conda index e51ca487e760..a057399d3dbe 100644 --- a/packages/pandas-gbq/ci/requirements-3.6-0.20.1.conda +++ b/packages/pandas-gbq/ci/requirements-3.6-0.20.1.conda @@ -1,3 +1,8 @@ google-auth google-auth-oauthlib -google-cloud-bigquery +google-cloud-bigquery==0.32.0 +pytest +pytest-cov +codecov +coverage +flake8 diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index b7febafd9873..ae1d35c1d734 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -20,6 +20,7 @@ def mock_bigquery_client(monkeypatch): mock_client = mock.create_autospec(google.cloud.bigquery.Client) # Mock out SELECT 1 query results. mock_query = mock.create_autospec(google.cloud.bigquery.QueryJob) + mock_query.job_id = 'some-random-id' mock_query.state = 'DONE' mock_rows = mock.create_autospec( google.cloud.bigquery.table.RowIterator) From 4976811cf3f5d5977a4d0643945937ec2e284d5e Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+maxim-lian@users.noreply.github.com> Date: Wed, 25 Apr 2018 15:41:00 -0400 Subject: [PATCH 126/519] ENH: project_id optional for to_gbq and read_gbq (#127) * project_id is optional * don't skip if no project * docstring & import order * project not required only if default creds available * Use tuple for credentials & project for default project detection. * Update bad_project_id test to query actual data. I think BigQuery stopped checking for valid project on queries with no data access. * Skip credentials tests if key not present. --- packages/pandas-gbq/.gitignore | 3 +- packages/pandas-gbq/docs/source/changelog.rst | 7 ++ packages/pandas-gbq/pandas_gbq/gbq.py | 43 +++++---- packages/pandas-gbq/tests/system.py | 88 +++++++------------ packages/pandas-gbq/tests/unit/test_gbq.py | 44 ++++++---- 5 files changed, 95 insertions(+), 90 deletions(-) diff --git a/packages/pandas-gbq/.gitignore b/packages/pandas-gbq/.gitignore index 33e26ed40d6e..251cc50df0ba 100644 --- a/packages/pandas-gbq/.gitignore +++ b/packages/pandas-gbq/.gitignore @@ -20,7 +20,8 @@ .ipynb_checkpoints .tags .pytest_cache -.testmondata +.testmon* +.vscode/ # Docs # ######## diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index a454ec84a300..5d7d4dd7afd1 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -4,6 +4,13 @@ Changelog 0.5.0 / TBD ----------- +- Project ID parameter is optional in ``read_gbq`` and ``to_gbq`` when it can + inferred from the environment. Note: you must still pass in a project ID when + using user-based authentication. (:issue:`103`) + +Internal changes +~~~~~~~~~~~~~~~~ + - Tests now use `nox` to run in multiple Python environments. (:issue:`52`) - Renamed internal modules. (:issue:`154`) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 5c8af053abe9..b74470746aad 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -186,7 +186,15 @@ def __init__(self, project_id, reauth=False, self.auth_local_webserver = auth_local_webserver self.dialect = dialect self.credentials_path = _get_credentials_file() - self.credentials = self.get_credentials() + self.credentials, default_project = self.get_credentials() + + if self.project_id is None: + self.project_id = default_project + + if self.project_id is None: + raise ValueError( + 'Could not determine project ID and one was not supplied.') + self.client = self.get_client() # BQ Queries costs $5 per TB. First 1 TB per month is free @@ -196,12 +204,14 @@ def __init__(self, project_id, reauth=False, def get_credentials(self): if self.private_key: return self.get_service_account_credentials() - else: - # Try to retrieve Application Default Credentials - credentials = self.get_application_default_credentials() - if not credentials: - credentials = self.get_user_account_credentials() - return credentials + + # Try to retrieve Application Default Credentials + credentials, default_project = ( + self.get_application_default_credentials()) + if credentials: + return credentials, default_project + + return self.get_user_account_credentials(), None def get_application_default_credentials(self): """ @@ -227,11 +237,13 @@ def get_application_default_credentials(self): from google.auth.exceptions import DefaultCredentialsError try: - credentials, _ = google.auth.default(scopes=[self.scope]) + credentials, default_project = google.auth.default( + scopes=[self.scope]) except (DefaultCredentialsError, IOError): - return None + return None, None - return _try_credentials(self.project_id, credentials) + billing_project = self.project_id or default_project + return _try_credentials(billing_project, credentials), default_project def load_user_account_credentials(self): """ @@ -412,7 +424,7 @@ def get_service_account_credentials(self): request = google.auth.transport.requests.Request() credentials.refresh(request) - return credentials + return credentials, json_key.get('project_id') except (KeyError, ValueError, TypeError, AttributeError): raise InvalidPrivateKeyFormat( "Private key is missing or invalid. It should be service " @@ -750,7 +762,7 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, ---------- query : str SQL-Like Query to return data values - project_id : str + project_id : str (optional when available in environment) Google BigQuery Account project ID. index_col : str (optional) Name of result column to use for index in results DataFrame @@ -809,9 +821,6 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, "a future version. Set logging level in order to vary " "verbosity", FutureWarning, stacklevel=1) - if not project_id: - raise TypeError("Missing required parameter: project_id") - if dialect not in ('legacy', 'standard'): raise ValueError("'{0}' is not valid for dialect".format(dialect)) @@ -859,7 +868,7 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, return final_df -def to_gbq(dataframe, destination_table, project_id, chunksize=None, +def to_gbq(dataframe, destination_table, project_id=None, chunksize=None, verbose=None, reauth=False, if_exists='fail', private_key=None, auth_local_webserver=False, table_schema=None): """Write a DataFrame to a Google BigQuery table. @@ -891,7 +900,7 @@ def to_gbq(dataframe, destination_table, project_id, chunksize=None, DataFrame to be written destination_table : string Name of table to be written, in the form 'dataset.tablename' - project_id : str + project_id : str (optional when available in environment) Google BigQuery Account project ID. chunksize : int (default None) Number of rows to be inserted in each chunk from the dataframe. Use diff --git a/packages/pandas-gbq/tests/system.py b/packages/pandas-gbq/tests/system.py index 8782b5801f98..6f57df3e75a6 100644 --- a/packages/pandas-gbq/tests/system.py +++ b/packages/pandas-gbq/tests/system.py @@ -50,12 +50,8 @@ def _get_dataset_prefix_random(): def _get_project_id(): - - project = os.environ.get('GBQ_PROJECT_ID') - if not project: - pytest.skip( - "Cannot run integration tests without a project id") - return project + return (os.environ.get('GBQ_PROJECT_ID') + or os.environ.get('GOOGLE_CLOUD_PROJECT')) # noqa def _get_private_key_path(): @@ -85,9 +81,12 @@ def _test_imports(): gbq._test_google_api_imports() -@pytest.fixture -def project(): - return _get_project_id() +@pytest.fixture(params=['env']) +def project(request): + if request.param == 'env': + return _get_project_id() + elif request.param == 'none': + return None def _check_if_can_get_correct_default_credentials(): @@ -99,11 +98,13 @@ def _check_if_can_get_correct_default_credentials(): from google.auth.exceptions import DefaultCredentialsError try: - credentials, _ = google.auth.default(scopes=[gbq.GbqConnector.scope]) + credentials, project = google.auth.default( + scopes=[gbq.GbqConnector.scope]) except (DefaultCredentialsError, IOError): return False - return gbq._try_credentials(_get_project_id(), credentials) is not None + return gbq._try_credentials( + project or _get_project_id(), credentials) is not None def clean_gbq_environment(dataset_prefix, private_key=None): @@ -171,46 +172,14 @@ def test_generate_bq_schema_deprecated(): gbq.generate_bq_schema(df) -@pytest.fixture(params=['local', 'service_path', 'service_creds']) -def auth_type(request): - - auth = request.param - - if auth == 'local': - - if _in_travis_environment(): - pytest.skip("Cannot run local auth in travis environment") - - elif auth == 'service_path': - - if _in_travis_environment(): - pytest.skip("Only run one auth type in Travis to save time") - - _skip_if_no_private_key_path() - elif auth == 'service_creds': - _skip_if_no_private_key_contents() - else: - raise ValueError - return auth - - @pytest.fixture() -def credentials(auth_type): - - if auth_type == 'local': - return None - - elif auth_type == 'service_path': - return _get_private_key_path() - elif auth_type == 'service_creds': - return _get_private_key_contents() - else: - raise ValueError +def credentials(): + _skip_if_no_private_key_contents() + return _get_private_key_contents() @pytest.fixture() def gbq_connector(project, credentials): - return gbq.GbqConnector(project, private_key=credentials) @@ -220,7 +189,7 @@ def test_should_be_able_to_make_a_connector(self, gbq_connector): assert gbq_connector is not None, 'Could not create a GbqConnector' def test_should_be_able_to_get_valid_credentials(self, gbq_connector): - credentials = gbq_connector.get_credentials() + credentials, _ = gbq_connector.get_credentials() assert credentials.valid def test_should_be_able_to_get_a_bigquery_client(self, gbq_connector): @@ -236,14 +205,12 @@ def test_should_be_able_to_get_results_from_query(self, gbq_connector): assert pages is not None -class TestGBQConnectorIntegrationWithLocalUserAccountAuth(object): +class TestAuth(object): @pytest.fixture(autouse=True) - def setup(self, project): - - _skip_local_auth_if_in_travis_env() - - self.sut = gbq.GbqConnector(project, auth_local_webserver=True) + def setup(self, gbq_connector): + self.sut = gbq_connector + self.sut.auth_local_webserver = True def test_get_application_default_credentials_does_not_throw_error(self): if _check_if_can_get_correct_default_credentials(): @@ -252,9 +219,9 @@ def test_get_application_default_credentials_does_not_throw_error(self): from google.auth.exceptions import DefaultCredentialsError with mock.patch('google.auth.default', side_effect=DefaultCredentialsError()): - credentials = self.sut.get_application_default_credentials() + credentials, _ = self.sut.get_application_default_credentials() else: - credentials = self.sut.get_application_default_credentials() + credentials, _ = self.sut.get_application_default_credentials() assert credentials is None def test_get_application_default_credentials_returns_credentials(self): @@ -262,10 +229,14 @@ def test_get_application_default_credentials_returns_credentials(self): pytest.skip("Cannot get default_credentials " "from the environment!") from google.auth.credentials import Credentials - credentials = self.sut.get_application_default_credentials() + credentials, default_project = ( + self.sut.get_application_default_credentials()) + assert isinstance(credentials, Credentials) + assert default_project is not None def test_get_user_account_credentials_bad_file_returns_credentials(self): + _skip_local_auth_if_in_travis_env() from google.auth.credentials import Credentials with mock.patch('__main__.open', side_effect=IOError()): @@ -273,6 +244,8 @@ def test_get_user_account_credentials_bad_file_returns_credentials(self): assert isinstance(credentials, Credentials) def test_get_user_account_credentials_returns_credentials(self): + _skip_local_auth_if_in_travis_env() + from google.auth.credentials import Credentials credentials = self.sut.get_user_account_credentials() assert isinstance(credentials, Credentials) @@ -515,7 +488,8 @@ def test_malformed_query(self): def test_bad_project_id(self): with pytest.raises(gbq.GenericGBQException): - gbq.read_gbq("SELECT 1", project_id='001', + gbq.read_gbq('SELCET * FROM [publicdata:samples.shakespeare]', + project_id='not-my-project', private_key=self.credentials) def test_bad_table_name(self): diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index ae1d35c1d734..85e4f427af73 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -15,9 +15,13 @@ @pytest.fixture(autouse=True) def mock_bigquery_client(monkeypatch): + from google.api_core.exceptions import NotFound import google.cloud.bigquery import google.cloud.bigquery.table mock_client = mock.create_autospec(google.cloud.bigquery.Client) + mock_schema = [ + google.cloud.bigquery.SchemaField('_f0', 'INTEGER') + ] # Mock out SELECT 1 query results. mock_query = mock.create_autospec(google.cloud.bigquery.QueryJob) mock_query.job_id = 'some-random-id' @@ -25,11 +29,12 @@ def mock_bigquery_client(monkeypatch): mock_rows = mock.create_autospec( google.cloud.bigquery.table.RowIterator) mock_rows.total_rows = 1 - mock_rows.schema = [ - google.cloud.bigquery.SchemaField('_f0', 'INTEGER')] + mock_rows.schema = mock_schema mock_rows.__iter__.return_value = [(1,)] mock_query.result.return_value = mock_rows mock_client.query.return_value = mock_query + # Mock table creation. + mock_client.get_table.side_effect = NotFound('nope') monkeypatch.setattr( gbq.GbqConnector, 'get_client', lambda _: mock_client) @@ -42,11 +47,7 @@ def no_auth(monkeypatch): monkeypatch.setattr( gbq.GbqConnector, 'get_application_default_credentials', - lambda _: mock_credentials) - monkeypatch.setattr( - gbq.GbqConnector, - 'get_user_account_credentials', - lambda _: mock_credentials) + lambda _: (mock_credentials, 'default-project')) def test_should_return_credentials_path_set_by_env_var(): @@ -76,12 +77,16 @@ def test_should_return_bigquery_correctly_typed( def test_to_gbq_should_fail_if_invalid_table_name_passed(): with pytest.raises(gbq.NotFoundException): - gbq.to_gbq(DataFrame(), 'invalid_table_name', project_id="1234") + gbq.to_gbq(DataFrame([[1]]), 'invalid_table_name', project_id="1234") -def test_to_gbq_with_no_project_id_given_should_fail(): +def test_to_gbq_with_no_project_id_given_should_fail(monkeypatch): + monkeypatch.setattr( + gbq.GbqConnector, + 'get_application_default_credentials', + lambda _: None) with pytest.raises(TypeError): - gbq.to_gbq(DataFrame(), 'dataset.tablename') + gbq.to_gbq(DataFrame([[1]]), 'dataset.tablename') def test_to_gbq_with_verbose_new_pandas_warns_deprecation(): @@ -95,7 +100,7 @@ def test_to_gbq_with_verbose_new_pandas_warns_deprecation(): mock_version.side_effect = [min_bq_version, pandas_version] try: gbq.to_gbq( - DataFrame(), + DataFrame([[1]]), 'dataset.tablename', project_id='my-project', verbose=True) @@ -114,7 +119,7 @@ def test_to_gbq_with_not_verbose_new_pandas_warns_deprecation(): mock_version.side_effect = [min_bq_version, pandas_version] try: gbq.to_gbq( - DataFrame(), + DataFrame([[1]]), 'dataset.tablename', project_id='my-project', verbose=False) @@ -132,7 +137,7 @@ def test_to_gbq_wo_verbose_w_new_pandas_no_warnings(recwarn): mock_version.side_effect = [min_bq_version, pandas_version] try: gbq.to_gbq( - DataFrame(), 'dataset.tablename', project_id='my-project') + DataFrame([[1]]), 'dataset.tablename', project_id='my-project') except gbq.TableCreationError: pass assert len(recwarn) == 0 @@ -148,7 +153,7 @@ def test_to_gbq_with_verbose_old_pandas_no_warnings(recwarn): mock_version.side_effect = [min_bq_version, pandas_version] try: gbq.to_gbq( - DataFrame(), + DataFrame([[1]]), 'dataset.tablename', project_id='my-project', verbose=True) @@ -157,11 +162,20 @@ def test_to_gbq_with_verbose_old_pandas_no_warnings(recwarn): assert len(recwarn) == 0 -def test_read_gbq_with_no_project_id_given_should_fail(): +def test_read_gbq_with_no_project_id_given_should_fail(monkeypatch): + monkeypatch.setattr( + gbq.GbqConnector, + 'get_application_default_credentials', + lambda _: None) with pytest.raises(TypeError): gbq.read_gbq('SELECT 1') +def test_read_gbq_with_inferred_project_id(monkeypatch): + df = gbq.read_gbq('SELECT 1') + assert df is not None + + def test_that_parse_data_works_properly(): from google.cloud.bigquery.table import Row test_schema = {'fields': [ From 376e823692ea11f35df148631f70f0a2004fd30d Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Fri, 27 Apr 2018 22:17:34 +0100 Subject: [PATCH 127/519] Add progress for to_gbq function (#166) * Add progress for to_gbq function using tqdm * Add tqdm addition to changelog * Add issue number to changelog for progress bar --- packages/pandas-gbq/docs/source/changelog.rst | 3 +++ packages/pandas-gbq/pandas_gbq/gbq.py | 22 ++++++++++++++----- packages/pandas-gbq/requirements.txt | 1 + packages/pandas-gbq/setup.py | 4 ++++ 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 5d7d4dd7afd1..7864d81e0e43 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -7,6 +7,9 @@ Changelog - Project ID parameter is optional in ``read_gbq`` and ``to_gbq`` when it can inferred from the environment. Note: you must still pass in a project ID when using user-based authentication. (:issue:`103`) +- Progress bar added for ``to_gbq``, through an optional library `tqdm` as + dependency. (:issue:`162`) + Internal changes ~~~~~~~~~~~~~~~~ diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index b74470746aad..5da85e82da39 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -16,6 +16,11 @@ BIGQUERY_INSTALLED_VERSION = None SHOW_VERBOSE_DEPRECATION = False +try: + import tqdm # noqa +except ImportError: + tqdm = None + def _check_google_client_version(): global BIGQUERY_INSTALLED_VERSION, SHOW_VERBOSE_DEPRECATION @@ -563,16 +568,19 @@ def run_query(self, query, **kwargs): def load_data( self, dataframe, dataset_id, table_id, chunksize=None, - schema=None): + schema=None, progress_bar=True): from pandas_gbq import load total_rows = len(dataframe) logger.info("\n\n") try: - for remaining_rows in load.load_chunks( - self.client, dataframe, dataset_id, table_id, - chunksize=chunksize, schema=schema): + chunks = load.load_chunks(self.client, dataframe, dataset_id, + table_id, chunksize=chunksize, + schema=schema) + if progress_bar and tqdm: + chunks = tqdm.tqdm(chunks) + for remaining_rows in chunks: logger.info("\rLoad is {0}% Complete".format( ((total_rows - remaining_rows) * 100) / total_rows)) except self.http_error as ex: @@ -870,7 +878,7 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, def to_gbq(dataframe, destination_table, project_id=None, chunksize=None, verbose=None, reauth=False, if_exists='fail', private_key=None, - auth_local_webserver=False, table_schema=None): + auth_local_webserver=False, table_schema=None, progress_bar=True): """Write a DataFrame to a Google BigQuery table. The main method a user calls to export pandas DataFrame contents to @@ -935,6 +943,8 @@ def to_gbq(dataframe, destination_table, project_id=None, chunksize=None, names of a field. .. versionadded:: 0.3.1 verbose : None, deprecated + progress_bar : boolean, True by default. It uses the library `tqdm` to show + the progress bar for the upload, chunk by chunk. """ _test_google_api_imports() @@ -987,7 +997,7 @@ def to_gbq(dataframe, destination_table, project_id=None, chunksize=None, connector.load_data( dataframe, dataset_id, table_id, chunksize=chunksize, - schema=table_schema) + schema=table_schema, progress_bar=progress_bar) def generate_bq_schema(df, default_type='STRING'): diff --git a/packages/pandas-gbq/requirements.txt b/packages/pandas-gbq/requirements.txt index 88cf967a0b42..7b3ede9750ae 100644 --- a/packages/pandas-gbq/requirements.txt +++ b/packages/pandas-gbq/requirements.txt @@ -2,3 +2,4 @@ pandas google-auth google-auth-oauthlib google-cloud-bigquery +tqdm diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index ebe147c30e69..40cfa42765b8 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -24,6 +24,9 @@ def readme(): 'google-cloud-bigquery>=0.29.0', ] +extras = { + 'tqdm': 'tqdm>=4.23.0', +} setup( name=NAME, @@ -50,6 +53,7 @@ def readme(): ], keywords='data', install_requires=INSTALL_REQUIRES, + extras_require=extras, packages=find_packages(exclude=['contrib', 'docs', 'tests*']), test_suite='tests', ) From a859ca4a9fefcc19ed66d2976c5bafccb4ebb834 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 4 May 2018 12:15:10 -0700 Subject: [PATCH 128/519] TST: use latest version of miniconda on Travis (#172) My local tests seem to show that https://github.com/conda-forge/google-cloud-bigquery-feedstock/issues/10 has been resolved with the latest release of Miniconda (most recent update was on 2018-05-02). --- packages/pandas-gbq/.travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/.travis.yml b/packages/pandas-gbq/.travis.yml index b30f79aa8e69..74380ccc54aa 100644 --- a/packages/pandas-gbq/.travis.yml +++ b/packages/pandas-gbq/.travis.yml @@ -25,7 +25,7 @@ install: if [ -f "$REQ.pip" ]; then pip install --upgrade nox-automation ; else - wget http://repo.continuum.io/miniconda/Miniconda3-4.3.30-Linux-x86_64.sh -O miniconda.sh; + wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; bash miniconda.sh -b -p $HOME/miniconda ; export PATH="$HOME/miniconda/bin:$PATH" ; hash -r ; @@ -49,7 +49,7 @@ script: - if [[ $PYTHON == '3.6' ]] && [[ "$PANDAS" == "MASTER" ]]; then nox -s test36master ; fi - REQ="ci/requirements-${PYTHON}-${PANDAS}" ; if [ -f "$REQ.conda" ]; then - pytest -v tests ; + pytest -v tests/unit tests/system.py ; fi - if [[ $COVERAGE == 'true' ]]; then nox -s cover ; fi - if [[ $LINT == 'true' ]]; then nox -s lint ; fi From f92e3c09b078a12cdcd88da7880b143fccf4e6c5 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 4 May 2018 12:15:29 -0700 Subject: [PATCH 129/519] CLN: remove workaround for streaming delays. (#173) Now that load jobs are used, no sleep is required when replacing a table. --- packages/pandas-gbq/pandas_gbq/gbq.py | 15 --------------- packages/pandas-gbq/tests/system.py | 11 ----------- 2 files changed, 26 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 5da85e82da39..b44cca0c939e 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -4,7 +4,6 @@ import time import warnings from datetime import datetime -from time import sleep import numpy as np from pandas import DataFrame, compat @@ -690,24 +689,10 @@ def schema_is_subset(self, dataset_id, table_id, schema): return all(field in fields_remote for field in fields_local) def delete_and_recreate_table(self, dataset_id, table_id, table_schema): - delay = 0 - - # Changes to table schema may take up to 2 minutes as of May 2015 See - # `Issue 191 - # `__ - # Compare previous schema with new schema to determine if there should - # be a 120 second delay - - if not self.verify_schema(dataset_id, table_id, table_schema): - logger.info('The existing table has a different schema. Please ' - 'wait 2 minutes. See Google BigQuery issue #191') - delay = 120 - table = _Table(self.project_id, dataset_id, private_key=self.private_key) table.delete(table_id) table.create(table_id, table_schema) - sleep(delay) def _get_credentials_file(): diff --git a/packages/pandas-gbq/tests/system.py b/packages/pandas-gbq/tests/system.py index 6f57df3e75a6..4042dd4bd014 100644 --- a/packages/pandas-gbq/tests/system.py +++ b/packages/pandas-gbq/tests/system.py @@ -872,17 +872,6 @@ def test_upload_subset_columns_if_table_exists_append(self): private_key=self.credentials) assert result['num_rows'][0] == test_size * 2 - # This test is currently failing intermittently due to changes in the - # BigQuery backend. You can track the issue in the Google BigQuery issue - # tracker `here `__. - # Currently you need to stream data twice in order to successfully stream - # data when you delete and re-create a table with a different schema. - # Something to consider is that google-cloud-bigquery returns an array of - # streaming insert errors rather than raising an exception. In this - # scenario, a decision could be made by the user to check for streaming - # errors and retry as needed. See `Issue 75 - # `__ - @pytest.mark.xfail(reason="Delete/create table w/ different schema issue") def test_upload_data_if_table_exists_replace(self): test_id = "4" test_size = 10 From 46f4b32ad426729b030d6f3d6d6b616bea481e11 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 7 May 2018 09:09:18 -0700 Subject: [PATCH 130/519] CLN: refactor auth logic to its own module. (#176) * CLN: refactor auth logic to its own module. Uses new-style tests for auth-related tests. * Mock new auth module functions in gbq unit tests. * Use monkeypatch in unit tests to not break system tests. * Use private_key and explicit project when default creds not available (such as on Travis) * Use explicit project with default credentials if passed in * Add comment why check default auth credentials. * Skip auth tests when key not available. * Move generate gbq schema test to unit tests. * Use pytest marks to skip local auth tests on Travis. * Skip tests when no private key file. * Use shared test fixtures for auth and gbq system tests. --- packages/pandas-gbq/.travis.yml | 2 +- packages/pandas-gbq/nox.py | 15 +- packages/pandas-gbq/pandas_gbq/auth.py | 280 +++++++++++ packages/pandas-gbq/pandas_gbq/exceptions.py | 14 + packages/pandas-gbq/pandas_gbq/gbq.py | 275 +---------- packages/pandas-gbq/tests/system/__init__.py | 0 packages/pandas-gbq/tests/system/conftest.py | 45 ++ packages/pandas-gbq/tests/system/test_auth.py | 101 ++++ .../tests/{system.py => system/test_gbq.py} | 443 +++++++----------- packages/pandas-gbq/tests/unit/test_gbq.py | 64 ++- 10 files changed, 669 insertions(+), 570 deletions(-) create mode 100644 packages/pandas-gbq/pandas_gbq/auth.py create mode 100644 packages/pandas-gbq/pandas_gbq/exceptions.py create mode 100644 packages/pandas-gbq/tests/system/__init__.py create mode 100644 packages/pandas-gbq/tests/system/conftest.py create mode 100644 packages/pandas-gbq/tests/system/test_auth.py rename packages/pandas-gbq/tests/{system.py => system/test_gbq.py} (77%) diff --git a/packages/pandas-gbq/.travis.yml b/packages/pandas-gbq/.travis.yml index 74380ccc54aa..cbd7695150c9 100644 --- a/packages/pandas-gbq/.travis.yml +++ b/packages/pandas-gbq/.travis.yml @@ -49,7 +49,7 @@ script: - if [[ $PYTHON == '3.6' ]] && [[ "$PANDAS" == "MASTER" ]]; then nox -s test36master ; fi - REQ="ci/requirements-${PYTHON}-${PANDAS}" ; if [ -f "$REQ.conda" ]; then - pytest -v tests/unit tests/system.py ; + pytest --quiet -m 'not local_auth' -v tests ; fi - if [[ $COVERAGE == 'true' ]]; then nox -s cover ; fi - if [[ $LINT == 'true' ]]; then nox -s lint ; fi diff --git a/packages/pandas-gbq/nox.py b/packages/pandas-gbq/nox.py index 7718b75c2029..53f3b8a6770a 100644 --- a/packages/pandas-gbq/nox.py +++ b/packages/pandas-gbq/nox.py @@ -3,6 +3,7 @@ See: https://nox.readthedocs.io/en/latest/ """ +import os import os.path import nox @@ -17,16 +18,24 @@ def default(session): session.install('mock', 'pytest', 'pytest-cov') session.install('-e', '.') + + # Skip local auth tests on Travis. + additional_args = list(session.posargs) + if 'TRAVIS_BUILD_DIR' in os.environ: + additional_args = additional_args + [ + '-m', + 'not local_auth', + ] + session.run( 'pytest', - os.path.join('.', 'tests', 'unit'), - os.path.join('.', 'tests', 'system.py'), + os.path.join('.', 'tests'), '--quiet', '--cov=pandas_gbq', '--cov=tests.unit', '--cov-report', 'xml:/tmp/pytest-cov.xml', - *session.posargs + *additional_args ) diff --git a/packages/pandas-gbq/pandas_gbq/auth.py b/packages/pandas-gbq/pandas_gbq/auth.py new file mode 100644 index 000000000000..f401243d09b2 --- /dev/null +++ b/packages/pandas-gbq/pandas_gbq/auth.py @@ -0,0 +1,280 @@ +"""Private module for fetching Google BigQuery credentials.""" + +import json +import logging +import os +import os.path + +import pandas.compat + +import pandas_gbq.exceptions + + +logger = logging.getLogger(__name__) + + +SCOPES = ['https://www.googleapis.com/auth/bigquery'] + + +def get_credentials( + private_key=None, project_id=None, reauth=False, + auth_local_webserver=False): + if private_key: + return get_service_account_credentials(private_key) + + # Try to retrieve Application Default Credentials + credentials, default_project = get_application_default_credentials( + project_id=project_id) + + if credentials: + return credentials, default_project + + credentials = get_user_account_credentials( + project_id=project_id, reauth=reauth, + auth_local_webserver=auth_local_webserver) + return credentials, project_id + + +def get_service_account_credentials(private_key): + import google.auth.transport.requests + from google.oauth2.service_account import Credentials + + try: + if os.path.isfile(private_key): + with open(private_key) as f: + json_key = json.loads(f.read()) + else: + # ugly hack: 'private_key' field has new lines inside, + # they break json parser, but we need to preserve them + json_key = json.loads(private_key.replace('\n', ' ')) + json_key['private_key'] = json_key['private_key'].replace( + ' ', '\n') + + if pandas.compat.PY3: + json_key['private_key'] = bytes( + json_key['private_key'], 'UTF-8') + + credentials = Credentials.from_service_account_info(json_key) + credentials = credentials.with_scopes(SCOPES) + + # Refresh the token before trying to use it. + request = google.auth.transport.requests.Request() + credentials.refresh(request) + + return credentials, json_key.get('project_id') + except (KeyError, ValueError, TypeError, AttributeError): + raise pandas_gbq.exceptions.InvalidPrivateKeyFormat( + "Private key is missing or invalid. It should be service " + "account private key JSON (file path or string contents) " + "with at least two keys: 'client_email' and 'private_key'. " + "Can be obtained from: https://console.developers.google." + "com/permissions/serviceaccounts") + + +def get_application_default_credentials(project_id=None): + """ + This method tries to retrieve the "default application credentials". + This could be useful for running code on Google Cloud Platform. + + Parameters + ---------- + project_id (str, optional): Override the default project ID. + + Returns + ------- + - GoogleCredentials, + If the default application credentials can be retrieved + from the environment. The retrieved credentials should also + have access to the project (project_id) on BigQuery. + - OR None, + If default application credentials can not be retrieved + from the environment. Or, the retrieved credentials do not + have access to the project (project_id) on BigQuery. + """ + import google.auth + from google.auth.exceptions import DefaultCredentialsError + + try: + credentials, default_project = google.auth.default(scopes=SCOPES) + except (DefaultCredentialsError, IOError): + return None, None + + # Even though we now have credentials, check that the credentials can be + # used with BigQuery. For example, we could be running on a GCE instance + # that does not allow the BigQuery scopes. + billing_project = project_id or default_project + return _try_credentials(billing_project, credentials), billing_project + + +def get_user_account_credentials( + project_id=None, reauth=False, auth_local_webserver=False, + credentials_path=None): + """Gets user account credentials. + + This method authenticates using user credentials, either loading saved + credentials from a file or by going through the OAuth flow. + + Parameters + ---------- + None + + Returns + ------- + GoogleCredentials : credentials + Credentials for the user with BigQuery access. + """ + from google_auth_oauthlib.flow import InstalledAppFlow + from oauthlib.oauth2.rfc6749.errors import OAuth2Error + + # Use the default credentials location under ~/.config and the + # equivalent directory on windows if the user has not specified a + # credentials path. + if not credentials_path: + credentials_path = get_default_credentials_path() + + # Previously, pandas-gbq saved user account credentials in the + # current working directory. If the bigquery_credentials.dat file + # exists in the current working directory, move the credentials to + # the new default location. + if os.path.isfile('bigquery_credentials.dat'): + os.rename('bigquery_credentials.dat', credentials_path) + + credentials = load_user_account_credentials( + project_id=project_id, credentials_path=credentials_path) + + client_config = { + 'installed': { + 'client_id': ('495642085510-k0tmvj2m941jhre2nbqka17vqpjfddtd' + '.apps.googleusercontent.com'), + 'client_secret': 'kOc9wMptUtxkcIFbtZCcrEAc', + 'redirect_uris': ['urn:ietf:wg:oauth:2.0:oob'], + 'auth_uri': 'https://accounts.google.com/o/oauth2/auth', + 'token_uri': 'https://accounts.google.com/o/oauth2/token', + } + } + + if credentials is None or reauth: + app_flow = InstalledAppFlow.from_client_config( + client_config, scopes=SCOPES) + + try: + if auth_local_webserver: + credentials = app_flow.run_local_server() + else: + credentials = app_flow.run_console() + except OAuth2Error as ex: + raise pandas_gbq.exceptions.AccessDenied( + "Unable to get valid credentials: {0}".format(ex)) + + save_user_account_credentials(credentials, credentials_path) + + return credentials + + +def load_user_account_credentials(project_id=None, credentials_path=None): + """ + Loads user account credentials from a local file. + + .. versionadded 0.2.0 + + Parameters + ---------- + None + + Returns + ------- + - GoogleCredentials, + If the credentials can loaded. The retrieved credentials should + also have access to the project (project_id) on BigQuery. + - OR None, + If credentials can not be loaded from a file. Or, the retrieved + credentials do not have access to the project (project_id) + on BigQuery. + """ + import google.auth.transport.requests + from google.oauth2.credentials import Credentials + + try: + with open(credentials_path) as credentials_file: + credentials_json = json.load(credentials_file) + except (IOError, ValueError): + return None + + credentials = Credentials( + token=credentials_json.get('access_token'), + refresh_token=credentials_json.get('refresh_token'), + id_token=credentials_json.get('id_token'), + token_uri=credentials_json.get('token_uri'), + client_id=credentials_json.get('client_id'), + client_secret=credentials_json.get('client_secret'), + scopes=credentials_json.get('scopes')) + + # Refresh the token before trying to use it. + request = google.auth.transport.requests.Request() + credentials.refresh(request) + + return _try_credentials(project_id, credentials) + + +def get_default_credentials_path(): + """ + Gets the default path to the BigQuery credentials + + .. versionadded 0.3.0 + + Returns + ------- + Path to the BigQuery credentials + """ + if os.name == 'nt': + config_path = os.environ['APPDATA'] + else: + config_path = os.path.join(os.path.expanduser('~'), '.config') + + config_path = os.path.join(config_path, 'pandas_gbq') + + # Create a pandas_gbq directory in an application-specific hidden + # user folder on the operating system. + if not os.path.exists(config_path): + os.makedirs(config_path) + + return os.path.join(config_path, 'bigquery_credentials.dat') + + +def save_user_account_credentials(credentials, credentials_path): + """ + Saves user account credentials to a local file. + + .. versionadded 0.2.0 + """ + try: + with open(credentials_path, 'w') as credentials_file: + credentials_json = { + 'refresh_token': credentials.refresh_token, + 'id_token': credentials.id_token, + 'token_uri': credentials.token_uri, + 'client_id': credentials.client_id, + 'client_secret': credentials.client_secret, + 'scopes': credentials.scopes, + } + json.dump(credentials_json, credentials_file) + except IOError: + logger.warning('Unable to save credentials.') + + +def _try_credentials(project_id, credentials): + from google.cloud import bigquery + import google.api_core.exceptions + + if not credentials: + return None + if not project_id: + return credentials + + try: + client = bigquery.Client(project=project_id, credentials=credentials) + # Check if the application has rights to the BigQuery project + client.query('SELECT 1').result() + return credentials + except google.api_core.exceptions.GoogleAPIError: + return None diff --git a/packages/pandas-gbq/pandas_gbq/exceptions.py b/packages/pandas-gbq/pandas_gbq/exceptions.py new file mode 100644 index 000000000000..a8b6aca0526e --- /dev/null +++ b/packages/pandas-gbq/pandas_gbq/exceptions.py @@ -0,0 +1,14 @@ + + +class AccessDenied(ValueError): + """ + Raised when invalid credentials are provided, or tokens have expired. + """ + pass + + +class InvalidPrivateKeyFormat(ValueError): + """ + Raised when provided private key has invalid format. + """ + pass diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index b44cca0c939e..0a14c7b19aa8 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -1,4 +1,4 @@ -import json + import logging import os import time @@ -6,9 +6,12 @@ from datetime import datetime import numpy as np -from pandas import DataFrame, compat +from pandas import DataFrame from pandas.compat import lzip +from pandas_gbq.exceptions import AccessDenied + + logger = logging.getLogger(__name__) @@ -73,36 +76,6 @@ def _test_google_api_imports(): _check_google_client_version() -def _try_credentials(project_id, credentials): - from google.cloud import bigquery - import google.api_core.exceptions - - if credentials is None: - return None - - try: - client = bigquery.Client(project=project_id, credentials=credentials) - # Check if the application has rights to the BigQuery project - client.query('SELECT 1').result() - return credentials - except google.api_core.exceptions.GoogleAPIError: - return None - - -class InvalidPrivateKeyFormat(ValueError): - """ - Raised when provided private key has invalid format. - """ - pass - - -class AccessDenied(ValueError): - """ - Raised when invalid credentials are provided, or tokens have expired. - """ - pass - - class DatasetCreationError(ValueError): """ Raised when the create dataset method fails @@ -176,13 +149,13 @@ class TableCreationError(ValueError): class GbqConnector(object): - scope = 'https://www.googleapis.com/auth/bigquery' def __init__(self, project_id, reauth=False, private_key=None, auth_local_webserver=False, dialect='legacy'): from google.api_core.exceptions import GoogleAPIError from google.api_core.exceptions import ClientError + from pandas_gbq import auth self.http_error = (ClientError, GoogleAPIError) self.project_id = project_id self.reauth = reauth @@ -190,7 +163,9 @@ def __init__(self, project_id, reauth=False, self.auth_local_webserver = auth_local_webserver self.dialect = dialect self.credentials_path = _get_credentials_file() - self.credentials, default_project = self.get_credentials() + self.credentials, default_project = auth.get_credentials( + private_key=private_key, project_id=project_id, reauth=reauth, + auth_local_webserver=auth_local_webserver) if self.project_id is None: self.project_id = default_project @@ -205,238 +180,6 @@ def __init__(self, project_id, reauth=False, # see here for more: https://cloud.google.com/bigquery/pricing self.query_price_for_TB = 5. / 2**40 # USD/TB - def get_credentials(self): - if self.private_key: - return self.get_service_account_credentials() - - # Try to retrieve Application Default Credentials - credentials, default_project = ( - self.get_application_default_credentials()) - if credentials: - return credentials, default_project - - return self.get_user_account_credentials(), None - - def get_application_default_credentials(self): - """ - This method tries to retrieve the "default application credentials". - This could be useful for running code on Google Cloud Platform. - - Parameters - ---------- - None - - Returns - ------- - - GoogleCredentials, - If the default application credentials can be retrieved - from the environment. The retrieved credentials should also - have access to the project (self.project_id) on BigQuery. - - OR None, - If default application credentials can not be retrieved - from the environment. Or, the retrieved credentials do not - have access to the project (self.project_id) on BigQuery. - """ - import google.auth - from google.auth.exceptions import DefaultCredentialsError - - try: - credentials, default_project = google.auth.default( - scopes=[self.scope]) - except (DefaultCredentialsError, IOError): - return None, None - - billing_project = self.project_id or default_project - return _try_credentials(billing_project, credentials), default_project - - def load_user_account_credentials(self): - """ - Loads user account credentials from a local file. - - .. versionadded 0.2.0 - - Parameters - ---------- - None - - Returns - ------- - - GoogleCredentials, - If the credentials can loaded. The retrieved credentials should - also have access to the project (self.project_id) on BigQuery. - - OR None, - If credentials can not be loaded from a file. Or, the retrieved - credentials do not have access to the project (self.project_id) - on BigQuery. - """ - import google.auth.transport.requests - from google.oauth2.credentials import Credentials - - # Use the default credentials location under ~/.config and the - # equivalent directory on windows if the user has not specified a - # credentials path. - if not self.credentials_path: - self.credentials_path = self.get_default_credentials_path() - - # Previously, pandas-gbq saved user account credentials in the - # current working directory. If the bigquery_credentials.dat file - # exists in the current working directory, move the credentials to - # the new default location. - if os.path.isfile('bigquery_credentials.dat'): - os.rename('bigquery_credentials.dat', self.credentials_path) - - try: - with open(self.credentials_path) as credentials_file: - credentials_json = json.load(credentials_file) - except (IOError, ValueError): - return None - - credentials = Credentials( - token=credentials_json.get('access_token'), - refresh_token=credentials_json.get('refresh_token'), - id_token=credentials_json.get('id_token'), - token_uri=credentials_json.get('token_uri'), - client_id=credentials_json.get('client_id'), - client_secret=credentials_json.get('client_secret'), - scopes=credentials_json.get('scopes')) - - # Refresh the token before trying to use it. - request = google.auth.transport.requests.Request() - credentials.refresh(request) - - return _try_credentials(self.project_id, credentials) - - def get_default_credentials_path(self): - """ - Gets the default path to the BigQuery credentials - - .. versionadded 0.3.0 - - Returns - ------- - Path to the BigQuery credentials - """ - - import os - - if os.name == 'nt': - config_path = os.environ['APPDATA'] - else: - config_path = os.path.join(os.path.expanduser('~'), '.config') - - config_path = os.path.join(config_path, 'pandas_gbq') - - # Create a pandas_gbq directory in an application-specific hidden - # user folder on the operating system. - if not os.path.exists(config_path): - os.makedirs(config_path) - - return os.path.join(config_path, 'bigquery_credentials.dat') - - def save_user_account_credentials(self, credentials): - """ - Saves user account credentials to a local file. - - .. versionadded 0.2.0 - """ - try: - with open(self.credentials_path, 'w') as credentials_file: - credentials_json = { - 'refresh_token': credentials.refresh_token, - 'id_token': credentials.id_token, - 'token_uri': credentials.token_uri, - 'client_id': credentials.client_id, - 'client_secret': credentials.client_secret, - 'scopes': credentials.scopes, - } - json.dump(credentials_json, credentials_file) - except IOError: - logger.warning('Unable to save credentials.') - - def get_user_account_credentials(self): - """Gets user account credentials. - - This method authenticates using user credentials, either loading saved - credentials from a file or by going through the OAuth flow. - - Parameters - ---------- - None - - Returns - ------- - GoogleCredentials : credentials - Credentials for the user with BigQuery access. - """ - from google_auth_oauthlib.flow import InstalledAppFlow - from oauthlib.oauth2.rfc6749.errors import OAuth2Error - - credentials = self.load_user_account_credentials() - - client_config = { - 'installed': { - 'client_id': ('495642085510-k0tmvj2m941jhre2nbqka17vqpjfddtd' - '.apps.googleusercontent.com'), - 'client_secret': 'kOc9wMptUtxkcIFbtZCcrEAc', - 'redirect_uris': ['urn:ietf:wg:oauth:2.0:oob'], - 'auth_uri': 'https://accounts.google.com/o/oauth2/auth', - 'token_uri': 'https://accounts.google.com/o/oauth2/token', - } - } - - if credentials is None or self.reauth: - app_flow = InstalledAppFlow.from_client_config( - client_config, scopes=[self.scope]) - - try: - if self.auth_local_webserver: - credentials = app_flow.run_local_server() - else: - credentials = app_flow.run_console() - except OAuth2Error as ex: - raise AccessDenied( - "Unable to get valid credentials: {0}".format(ex)) - - self.save_user_account_credentials(credentials) - - return credentials - - def get_service_account_credentials(self): - import google.auth.transport.requests - from google.oauth2.service_account import Credentials - from os.path import isfile - - try: - if isfile(self.private_key): - with open(self.private_key) as f: - json_key = json.loads(f.read()) - else: - # ugly hack: 'private_key' field has new lines inside, - # they break json parser, but we need to preserve them - json_key = json.loads(self.private_key.replace('\n', ' ')) - json_key['private_key'] = json_key['private_key'].replace( - ' ', '\n') - - if compat.PY3: - json_key['private_key'] = bytes( - json_key['private_key'], 'UTF-8') - - credentials = Credentials.from_service_account_info(json_key) - credentials = credentials.with_scopes([self.scope]) - - # Refresh the token before trying to use it. - request = google.auth.transport.requests.Request() - credentials.refresh(request) - - return credentials, json_key.get('project_id') - except (KeyError, ValueError, TypeError, AttributeError): - raise InvalidPrivateKeyFormat( - "Private key is missing or invalid. It should be service " - "account private key JSON (file path or string contents) " - "with at least two keys: 'client_email' and 'private_key'. " - "Can be obtained from: https://console.developers.google." - "com/permissions/serviceaccounts") - def _start_timer(self): self.start = time.time() diff --git a/packages/pandas-gbq/tests/system/__init__.py b/packages/pandas-gbq/tests/system/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/packages/pandas-gbq/tests/system/conftest.py b/packages/pandas-gbq/tests/system/conftest.py new file mode 100644 index 000000000000..18132e9aa316 --- /dev/null +++ b/packages/pandas-gbq/tests/system/conftest.py @@ -0,0 +1,45 @@ +"""Shared pytest fixtures for system tests.""" + +import os +import os.path + +import pytest + + +@pytest.fixture +def project_id(): + return (os.environ.get('GBQ_PROJECT_ID') + or os.environ.get('GOOGLE_CLOUD_PROJECT')) # noqa + + +@pytest.fixture +def private_key_path(): + path = None + if 'TRAVIS_BUILD_DIR' in os.environ: + path = os.path.join( + os.environ['TRAVIS_BUILD_DIR'], 'ci', + 'travis_gbq.json') + elif 'GBQ_GOOGLE_APPLICATION_CREDENTIALS' in os.environ: + path = os.environ['GBQ_GOOGLE_APPLICATION_CREDENTIALS'] + elif 'GOOGLE_APPLICATION_CREDENTIALS' in os.environ: + path = os.environ['GOOGLE_APPLICATION_CREDENTIALS'] + + if path is None: + pytest.skip("Cannot run integration tests without a " + "private key json file path") + return None + if not os.path.isfile(path): + pytest.skip("Cannot run integration tests when there is " + "no file at the private key json file path") + return None + + return path + + +@pytest.fixture +def private_key_contents(private_key_path): + if private_key_path is None: + return None + + with open(private_key_path) as f: + return f.read() diff --git a/packages/pandas-gbq/tests/system/test_auth.py b/packages/pandas-gbq/tests/system/test_auth.py new file mode 100644 index 000000000000..c75a28d54e87 --- /dev/null +++ b/packages/pandas-gbq/tests/system/test_auth.py @@ -0,0 +1,101 @@ +"""System tests for fetching Google BigQuery credentials.""" + +try: + import mock +except ImportError: # pragma: NO COVER + from unittest import mock +import pytest + +from pandas_gbq import auth + + +def _check_if_can_get_correct_default_credentials(): + # Checks if "Application Default Credentials" can be fetched + # from the environment the tests are running in. + # See https://github.com/pandas-dev/pandas/issues/13577 + + import google.auth + from google.auth.exceptions import DefaultCredentialsError + import pandas_gbq.auth + import pandas_gbq.gbq + + try: + credentials, project = google.auth.default( + scopes=pandas_gbq.auth.SCOPES) + except (DefaultCredentialsError, IOError): + return False + + return auth._try_credentials(project, credentials) is not None + + +def test_should_be_able_to_get_valid_credentials(project_id, private_key_path): + credentials, _ = auth.get_credentials( + project_id=project_id, private_key=private_key_path) + assert credentials.valid + + +def test_get_service_account_credentials_private_key_path(private_key_path): + from google.auth.credentials import Credentials + credentials, project_id = auth.get_service_account_credentials( + private_key_path) + assert isinstance(credentials, Credentials) + assert auth._try_credentials(project_id, credentials) is not None + + +def test_get_service_account_credentials_private_key_contents( + private_key_contents): + from google.auth.credentials import Credentials + credentials, project_id = auth.get_service_account_credentials( + private_key_contents) + assert isinstance(credentials, Credentials) + assert auth._try_credentials(project_id, credentials) is not None + + +def test_get_application_default_credentials_does_not_throw_error(): + if _check_if_can_get_correct_default_credentials(): + # Can get real credentials, so mock it out to fail. + from google.auth.exceptions import DefaultCredentialsError + with mock.patch('google.auth.default', + side_effect=DefaultCredentialsError()): + credentials, _ = auth.get_application_default_credentials() + else: + credentials, _ = auth.get_application_default_credentials() + assert credentials is None + + +def test_get_application_default_credentials_returns_credentials(): + if not _check_if_can_get_correct_default_credentials(): + pytest.skip("Cannot get default_credentials " + "from the environment!") + from google.auth.credentials import Credentials + credentials, default_project = auth.get_application_default_credentials() + + assert isinstance(credentials, Credentials) + assert default_project is not None + + +@pytest.mark.local_auth +def test_get_user_account_credentials_bad_file_returns_credentials(): + from google.auth.credentials import Credentials + with mock.patch('__main__.open', side_effect=IOError()): + credentials = auth.get_user_account_credentials() + assert isinstance(credentials, Credentials) + + +@pytest.mark.local_auth +def test_get_user_account_credentials_returns_credentials(project_id): + from google.auth.credentials import Credentials + credentials = auth.get_user_account_credentials( + project_id=project_id, + auth_local_webserver=True) + assert isinstance(credentials, Credentials) + + +@pytest.mark.local_auth +def test_get_user_account_credentials_reauth_returns_credentials(project_id): + from google.auth.credentials import Credentials + credentials = auth.get_user_account_credentials( + project_id=project_id, + auth_local_webserver=True, + reauth=True) + assert isinstance(credentials, Credentials) diff --git a/packages/pandas-gbq/tests/system.py b/packages/pandas-gbq/tests/system/test_gbq.py similarity index 77% rename from packages/pandas-gbq/tests/system.py rename to packages/pandas-gbq/tests/system/test_gbq.py index 4042dd4bd014..6cb792afe33c 100644 --- a/packages/pandas-gbq/tests/system.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -import os import sys from datetime import datetime from random import randint @@ -15,62 +14,14 @@ from pandas_gbq import gbq -try: - import mock -except ImportError: # pragma: NO COVER - from unittest import mock TABLE_ID = 'new_test' -def _skip_local_auth_if_in_travis_env(): - if _in_travis_environment(): - pytest.skip("Cannot run local auth in travis environment") - - -def _skip_if_no_private_key_path(): - if not _get_private_key_path(): - pytest.skip("Cannot run integration tests without a " - "private key json file path") - - -def _skip_if_no_private_key_contents(): - if not _get_private_key_contents(): - raise pytest.skip("Cannot run integration tests without a " - "private key json contents") - - -def _in_travis_environment(): - return 'TRAVIS_BUILD_DIR' in os.environ and \ - 'GBQ_PROJECT_ID' in os.environ - - def _get_dataset_prefix_random(): return ''.join(['pandas_gbq_', str(randint(1, 100000))]) -def _get_project_id(): - return (os.environ.get('GBQ_PROJECT_ID') - or os.environ.get('GOOGLE_CLOUD_PROJECT')) # noqa - - -def _get_private_key_path(): - if _in_travis_environment(): - return os.path.join(*[os.environ.get('TRAVIS_BUILD_DIR'), 'ci', - 'travis_gbq.json']) - else: - return os.environ.get('GBQ_GOOGLE_APPLICATION_CREDENTIALS') - - -def _get_private_key_contents(): - key_path = _get_private_key_path() - if key_path is None: - return None - - with open(key_path) as f: - return f.read() - - @pytest.fixture(autouse=True, scope='module') def _test_imports(): try: @@ -82,33 +33,25 @@ def _test_imports(): @pytest.fixture(params=['env']) -def project(request): +def project(request, project_id): if request.param == 'env': - return _get_project_id() + return project_id elif request.param == 'none': return None -def _check_if_can_get_correct_default_credentials(): - # Checks if "Application Default Credentials" can be fetched - # from the environment the tests are running in. - # See https://github.com/pandas-dev/pandas/issues/13577 +@pytest.fixture() +def credentials(private_key_contents): + return private_key_contents - import google.auth - from google.auth.exceptions import DefaultCredentialsError - try: - credentials, project = google.auth.default( - scopes=[gbq.GbqConnector.scope]) - except (DefaultCredentialsError, IOError): - return False - - return gbq._try_credentials( - project or _get_project_id(), credentials) is not None +@pytest.fixture() +def gbq_connector(project, credentials): + return gbq.GbqConnector(project, private_key=credentials) -def clean_gbq_environment(dataset_prefix, private_key=None): - dataset = gbq._Dataset(_get_project_id(), private_key=private_key) +def clean_gbq_environment(dataset_prefix, private_key=None, project_id=None): + dataset = gbq._Dataset(project_id, private_key=private_key) all_datasets = dataset.datasets() retry = 3 @@ -118,7 +61,7 @@ def clean_gbq_environment(dataset_prefix, private_key=None): for i in range(1, 10): dataset_id = dataset_prefix + str(i) if dataset_id in all_datasets: - table = gbq._Table(_get_project_id(), dataset_id, + table = gbq._Table(project_id, dataset_id, private_key=private_key) # Table listing is eventually consistent, so loop until @@ -165,33 +108,11 @@ def make_mixed_dataframe_v2(test_size): index=range(test_size)) -def test_generate_bq_schema_deprecated(): - # 11121 Deprecation of generate_bq_schema - with pytest.warns(FutureWarning): - df = make_mixed_dataframe_v2(10) - gbq.generate_bq_schema(df) - - -@pytest.fixture() -def credentials(): - _skip_if_no_private_key_contents() - return _get_private_key_contents() - - -@pytest.fixture() -def gbq_connector(project, credentials): - return gbq.GbqConnector(project, private_key=credentials) - - class TestGBQConnectorIntegration(object): def test_should_be_able_to_make_a_connector(self, gbq_connector): assert gbq_connector is not None, 'Could not create a GbqConnector' - def test_should_be_able_to_get_valid_credentials(self, gbq_connector): - credentials, _ = gbq_connector.get_credentials() - assert credentials.valid - def test_should_be_able_to_get_a_bigquery_client(self, gbq_connector): bigquery_client = gbq_connector.get_client() assert bigquery_client is not None @@ -205,52 +126,6 @@ def test_should_be_able_to_get_results_from_query(self, gbq_connector): assert pages is not None -class TestAuth(object): - - @pytest.fixture(autouse=True) - def setup(self, gbq_connector): - self.sut = gbq_connector - self.sut.auth_local_webserver = True - - def test_get_application_default_credentials_does_not_throw_error(self): - if _check_if_can_get_correct_default_credentials(): - # Can get real credentials, so mock it out to fail. - - from google.auth.exceptions import DefaultCredentialsError - with mock.patch('google.auth.default', - side_effect=DefaultCredentialsError()): - credentials, _ = self.sut.get_application_default_credentials() - else: - credentials, _ = self.sut.get_application_default_credentials() - assert credentials is None - - def test_get_application_default_credentials_returns_credentials(self): - if not _check_if_can_get_correct_default_credentials(): - pytest.skip("Cannot get default_credentials " - "from the environment!") - from google.auth.credentials import Credentials - credentials, default_project = ( - self.sut.get_application_default_credentials()) - - assert isinstance(credentials, Credentials) - assert default_project is not None - - def test_get_user_account_credentials_bad_file_returns_credentials(self): - _skip_local_auth_if_in_travis_env() - - from google.auth.credentials import Credentials - with mock.patch('__main__.open', side_effect=IOError()): - credentials = self.sut.get_user_account_credentials() - assert isinstance(credentials, Credentials) - - def test_get_user_account_credentials_returns_credentials(self): - _skip_local_auth_if_in_travis_env() - - from google.auth.credentials import Credentials - credentials = self.sut.get_user_account_credentials() - assert isinstance(credentials, Credentials) - - def test_should_read(project, credentials): query = 'SELECT "PI" AS valid_string' df = gbq.read_gbq(query, project_id=project, private_key=credentials) @@ -268,152 +143,152 @@ def setup(self, project, credentials): project, private_key=credentials) self.credentials = credentials - def test_should_properly_handle_valid_strings(self): + def test_should_properly_handle_valid_strings(self, project_id): query = 'SELECT "PI" AS valid_string' - df = gbq.read_gbq(query, project_id=_get_project_id(), + df = gbq.read_gbq(query, project_id=project_id, private_key=self.credentials) tm.assert_frame_equal(df, DataFrame({'valid_string': ['PI']})) - def test_should_properly_handle_empty_strings(self): + def test_should_properly_handle_empty_strings(self, project_id): query = 'SELECT "" AS empty_string' - df = gbq.read_gbq(query, project_id=_get_project_id(), + df = gbq.read_gbq(query, project_id=project_id, private_key=self.credentials) tm.assert_frame_equal(df, DataFrame({'empty_string': [""]})) - def test_should_properly_handle_null_strings(self): + def test_should_properly_handle_null_strings(self, project_id): query = 'SELECT STRING(NULL) AS null_string' - df = gbq.read_gbq(query, project_id=_get_project_id(), + df = gbq.read_gbq(query, project_id=project_id, private_key=self.credentials) tm.assert_frame_equal(df, DataFrame({'null_string': [None]})) - def test_should_properly_handle_valid_integers(self): + def test_should_properly_handle_valid_integers(self, project_id): query = 'SELECT INTEGER(3) AS valid_integer' - df = gbq.read_gbq(query, project_id=_get_project_id(), + df = gbq.read_gbq(query, project_id=project_id, private_key=self.credentials) tm.assert_frame_equal(df, DataFrame({'valid_integer': [3]})) - def test_should_properly_handle_nullable_integers(self): + def test_should_properly_handle_nullable_integers(self, project_id): query = '''SELECT * FROM (SELECT 1 AS nullable_integer), (SELECT NULL AS nullable_integer)''' - df = gbq.read_gbq(query, project_id=_get_project_id(), + df = gbq.read_gbq(query, project_id=project_id, private_key=self.credentials) tm.assert_frame_equal( df, DataFrame({'nullable_integer': [1, None]}).astype(object)) - def test_should_properly_handle_valid_longs(self): + def test_should_properly_handle_valid_longs(self, project_id): query = 'SELECT 1 << 62 AS valid_long' - df = gbq.read_gbq(query, project_id=_get_project_id(), + df = gbq.read_gbq(query, project_id=project_id, private_key=self.credentials) tm.assert_frame_equal( df, DataFrame({'valid_long': [1 << 62]})) - def test_should_properly_handle_nullable_longs(self): + def test_should_properly_handle_nullable_longs(self, project_id): query = '''SELECT * FROM (SELECT 1 << 62 AS nullable_long), (SELECT NULL AS nullable_long)''' - df = gbq.read_gbq(query, project_id=_get_project_id(), + df = gbq.read_gbq(query, project_id=project_id, private_key=self.credentials) tm.assert_frame_equal( df, DataFrame({'nullable_long': [1 << 62, None]}).astype(object)) - def test_should_properly_handle_null_integers(self): + def test_should_properly_handle_null_integers(self, project_id): query = 'SELECT INTEGER(NULL) AS null_integer' - df = gbq.read_gbq(query, project_id=_get_project_id(), + df = gbq.read_gbq(query, project_id=project_id, private_key=self.credentials) tm.assert_frame_equal(df, DataFrame({'null_integer': [None]})) - def test_should_properly_handle_valid_floats(self): + def test_should_properly_handle_valid_floats(self, project_id): from math import pi query = 'SELECT PI() AS valid_float' - df = gbq.read_gbq(query, project_id=_get_project_id(), + df = gbq.read_gbq(query, project_id=project_id, private_key=self.credentials) tm.assert_frame_equal(df, DataFrame( {'valid_float': [pi]})) - def test_should_properly_handle_nullable_floats(self): + def test_should_properly_handle_nullable_floats(self, project_id): from math import pi query = '''SELECT * FROM (SELECT PI() AS nullable_float), (SELECT NULL AS nullable_float)''' - df = gbq.read_gbq(query, project_id=_get_project_id(), + df = gbq.read_gbq(query, project_id=project_id, private_key=self.credentials) tm.assert_frame_equal( df, DataFrame({'nullable_float': [pi, None]})) - def test_should_properly_handle_valid_doubles(self): + def test_should_properly_handle_valid_doubles(self, project_id): from math import pi query = 'SELECT PI() * POW(10, 307) AS valid_double' - df = gbq.read_gbq(query, project_id=_get_project_id(), + df = gbq.read_gbq(query, project_id=project_id, private_key=self.credentials) tm.assert_frame_equal(df, DataFrame( {'valid_double': [pi * 10 ** 307]})) - def test_should_properly_handle_nullable_doubles(self): + def test_should_properly_handle_nullable_doubles(self, project_id): from math import pi query = '''SELECT * FROM (SELECT PI() * POW(10, 307) AS nullable_double), (SELECT NULL AS nullable_double)''' - df = gbq.read_gbq(query, project_id=_get_project_id(), + df = gbq.read_gbq(query, project_id=project_id, private_key=self.credentials) tm.assert_frame_equal( df, DataFrame({'nullable_double': [pi * 10 ** 307, None]})) - def test_should_properly_handle_null_floats(self): + def test_should_properly_handle_null_floats(self, project_id): query = 'SELECT FLOAT(NULL) AS null_float' - df = gbq.read_gbq(query, project_id=_get_project_id(), + df = gbq.read_gbq(query, project_id=project_id, private_key=self.credentials) tm.assert_frame_equal(df, DataFrame({'null_float': [np.nan]})) - def test_should_properly_handle_timestamp_unix_epoch(self): + def test_should_properly_handle_timestamp_unix_epoch(self, project_id): query = 'SELECT TIMESTAMP("1970-01-01 00:00:00") AS unix_epoch' - df = gbq.read_gbq(query, project_id=_get_project_id(), + df = gbq.read_gbq(query, project_id=project_id, private_key=self.credentials) tm.assert_frame_equal(df, DataFrame( {'unix_epoch': [np.datetime64('1970-01-01T00:00:00.000000Z')]})) - def test_should_properly_handle_arbitrary_timestamp(self): + def test_should_properly_handle_arbitrary_timestamp(self, project_id): query = 'SELECT TIMESTAMP("2004-09-15 05:00:00") AS valid_timestamp' - df = gbq.read_gbq(query, project_id=_get_project_id(), + df = gbq.read_gbq(query, project_id=project_id, private_key=self.credentials) tm.assert_frame_equal(df, DataFrame({ 'valid_timestamp': [np.datetime64('2004-09-15T05:00:00.000000Z')] })) - def test_should_properly_handle_null_timestamp(self): + def test_should_properly_handle_null_timestamp(self, project_id): query = 'SELECT TIMESTAMP(NULL) AS null_timestamp' - df = gbq.read_gbq(query, project_id=_get_project_id(), + df = gbq.read_gbq(query, project_id=project_id, private_key=self.credentials) tm.assert_frame_equal(df, DataFrame({'null_timestamp': [NaT]})) - def test_should_properly_handle_true_boolean(self): + def test_should_properly_handle_true_boolean(self, project_id): query = 'SELECT BOOLEAN(TRUE) AS true_boolean' - df = gbq.read_gbq(query, project_id=_get_project_id(), + df = gbq.read_gbq(query, project_id=project_id, private_key=self.credentials) tm.assert_frame_equal(df, DataFrame({'true_boolean': [True]})) - def test_should_properly_handle_false_boolean(self): + def test_should_properly_handle_false_boolean(self, project_id): query = 'SELECT BOOLEAN(FALSE) AS false_boolean' - df = gbq.read_gbq(query, project_id=_get_project_id(), + df = gbq.read_gbq(query, project_id=project_id, private_key=self.credentials) tm.assert_frame_equal(df, DataFrame({'false_boolean': [False]})) - def test_should_properly_handle_null_boolean(self): + def test_should_properly_handle_null_boolean(self, project_id): query = 'SELECT BOOLEAN(NULL) AS null_boolean' - df = gbq.read_gbq(query, project_id=_get_project_id(), + df = gbq.read_gbq(query, project_id=project_id, private_key=self.credentials) tm.assert_frame_equal(df, DataFrame({'null_boolean': [None]})) - def test_should_properly_handle_nullable_booleans(self): + def test_should_properly_handle_nullable_booleans(self, project_id): query = '''SELECT * FROM (SELECT BOOLEAN(TRUE) AS nullable_boolean), (SELECT NULL AS nullable_boolean)''' - df = gbq.read_gbq(query, project_id=_get_project_id(), + df = gbq.read_gbq(query, project_id=project_id, private_key=self.credentials) tm.assert_frame_equal( df, DataFrame({'nullable_boolean': [True, None]}).astype(object)) - def test_unicode_string_conversion_and_normalization(self): + def test_unicode_string_conversion_and_normalization(self, project_id): correct_test_datatype = DataFrame( {'unicode_string': [u("\xe9\xfc")]} ) @@ -425,43 +300,43 @@ def test_unicode_string_conversion_and_normalization(self): query = 'SELECT "{0}" AS unicode_string'.format(unicode_string) - df = gbq.read_gbq(query, project_id=_get_project_id(), + df = gbq.read_gbq(query, project_id=project_id, private_key=self.credentials) tm.assert_frame_equal(df, correct_test_datatype) - def test_index_column(self): + def test_index_column(self, project_id): query = "SELECT 'a' AS string_1, 'b' AS string_2" - result_frame = gbq.read_gbq(query, project_id=_get_project_id(), + result_frame = gbq.read_gbq(query, project_id=project_id, index_col="string_1", private_key=self.credentials) correct_frame = DataFrame( {'string_1': ['a'], 'string_2': ['b']}).set_index("string_1") assert result_frame.index.name == correct_frame.index.name - def test_column_order(self): + def test_column_order(self, project_id): query = "SELECT 'a' AS string_1, 'b' AS string_2, 'c' AS string_3" col_order = ['string_3', 'string_1', 'string_2'] - result_frame = gbq.read_gbq(query, project_id=_get_project_id(), + result_frame = gbq.read_gbq(query, project_id=project_id, col_order=col_order, private_key=self.credentials) correct_frame = DataFrame({'string_1': ['a'], 'string_2': [ 'b'], 'string_3': ['c']})[col_order] tm.assert_frame_equal(result_frame, correct_frame) - def test_read_gbq_raises_invalid_column_order(self): + def test_read_gbq_raises_invalid_column_order(self, project_id): query = "SELECT 'a' AS string_1, 'b' AS string_2, 'c' AS string_3" col_order = ['string_aaa', 'string_1', 'string_2'] # Column string_aaa does not exist. Should raise InvalidColumnOrder with pytest.raises(gbq.InvalidColumnOrder): - gbq.read_gbq(query, project_id=_get_project_id(), + gbq.read_gbq(query, project_id=project_id, col_order=col_order, private_key=self.credentials) - def test_column_order_plus_index(self): + def test_column_order_plus_index(self, project_id): query = "SELECT 'a' AS string_1, 'b' AS string_2, 'c' AS string_3" col_order = ['string_3', 'string_2'] - result_frame = gbq.read_gbq(query, project_id=_get_project_id(), + result_frame = gbq.read_gbq(query, project_id=project_id, index_col='string_1', col_order=col_order, private_key=self.credentials) correct_frame = DataFrame( @@ -470,20 +345,20 @@ def test_column_order_plus_index(self): correct_frame = correct_frame[col_order] tm.assert_frame_equal(result_frame, correct_frame) - def test_read_gbq_raises_invalid_index_column(self): + def test_read_gbq_raises_invalid_index_column(self, project_id): query = "SELECT 'a' AS string_1, 'b' AS string_2, 'c' AS string_3" col_order = ['string_3', 'string_2'] # Column string_bbb does not exist. Should raise InvalidIndexColumn with pytest.raises(gbq.InvalidIndexColumn): - gbq.read_gbq(query, project_id=_get_project_id(), + gbq.read_gbq(query, project_id=project_id, index_col='string_bbb', col_order=col_order, private_key=self.credentials) - def test_malformed_query(self): + def test_malformed_query(self, project_id): with pytest.raises(gbq.GenericGBQException): gbq.read_gbq("SELCET * FORM [publicdata:samples.shakespeare]", - project_id=_get_project_id(), + project_id=project_id, private_key=self.credentials) def test_bad_project_id(self): @@ -492,30 +367,30 @@ def test_bad_project_id(self): project_id='not-my-project', private_key=self.credentials) - def test_bad_table_name(self): + def test_bad_table_name(self, project_id): with pytest.raises(gbq.GenericGBQException): gbq.read_gbq("SELECT * FROM [publicdata:samples.nope]", - project_id=_get_project_id(), + project_id=project_id, private_key=self.credentials) - def test_download_dataset_larger_than_200k_rows(self): + def test_download_dataset_larger_than_200k_rows(self, project_id): test_size = 200005 # Test for known BigQuery bug in datasets larger than 100k rows # http://stackoverflow.com/questions/19145587/bq-py-not-paging-results df = gbq.read_gbq("SELECT id FROM [publicdata:samples.wikipedia] " "GROUP EACH BY id ORDER BY id ASC LIMIT {0}" .format(test_size), - project_id=_get_project_id(), + project_id=project_id, private_key=self.credentials) assert len(df.drop_duplicates()) == test_size - def test_zero_rows(self): + def test_zero_rows(self, project_id): # Bug fix for https://github.com/pandas-dev/pandas/issues/10273 df = gbq.read_gbq("SELECT title, id, is_bot, " "SEC_TO_TIMESTAMP(timestamp) ts " "FROM [publicdata:samples.wikipedia] " "WHERE timestamp=-9999999", - project_id=_get_project_id(), + project_id=project_id, private_key=self.credentials) page_array = np.zeros( (0,), dtype=[('title', object), ('id', np.dtype(int)), @@ -524,56 +399,56 @@ def test_zero_rows(self): page_array, columns=['title', 'id', 'is_bot', 'ts']) tm.assert_frame_equal(df, expected_result) - def test_legacy_sql(self): + def test_legacy_sql(self, project_id): legacy_sql = "SELECT id FROM [publicdata.samples.wikipedia] LIMIT 10" # Test that a legacy sql statement fails when # setting dialect='standard' with pytest.raises(gbq.GenericGBQException): - gbq.read_gbq(legacy_sql, project_id=_get_project_id(), + gbq.read_gbq(legacy_sql, project_id=project_id, dialect='standard', private_key=self.credentials) # Test that a legacy sql statement succeeds when # setting dialect='legacy' - df = gbq.read_gbq(legacy_sql, project_id=_get_project_id(), + df = gbq.read_gbq(legacy_sql, project_id=project_id, dialect='legacy', private_key=self.credentials) assert len(df.drop_duplicates()) == 10 - def test_standard_sql(self): + def test_standard_sql(self, project_id): standard_sql = "SELECT DISTINCT id FROM " \ "`publicdata.samples.wikipedia` LIMIT 10" # Test that a standard sql statement fails when using # the legacy SQL dialect (default value) with pytest.raises(gbq.GenericGBQException): - gbq.read_gbq(standard_sql, project_id=_get_project_id(), + gbq.read_gbq(standard_sql, project_id=project_id, private_key=self.credentials) # Test that a standard sql statement succeeds when # setting dialect='standard' - df = gbq.read_gbq(standard_sql, project_id=_get_project_id(), + df = gbq.read_gbq(standard_sql, project_id=project_id, dialect='standard', private_key=self.credentials) assert len(df.drop_duplicates()) == 10 - def test_invalid_option_for_sql_dialect(self): + def test_invalid_option_for_sql_dialect(self, project_id): sql_statement = "SELECT DISTINCT id FROM " \ "`publicdata.samples.wikipedia` LIMIT 10" # Test that an invalid option for `dialect` raises ValueError with pytest.raises(ValueError): - gbq.read_gbq(sql_statement, project_id=_get_project_id(), + gbq.read_gbq(sql_statement, project_id=project_id, dialect='invalid', private_key=self.credentials) # Test that a correct option for dialect succeeds # to make sure ValueError was due to invalid dialect - gbq.read_gbq(sql_statement, project_id=_get_project_id(), + gbq.read_gbq(sql_statement, project_id=project_id, dialect='standard', private_key=self.credentials) - def test_query_with_parameters(self): + def test_query_with_parameters(self, project_id): sql_statement = "SELECT @param1 + @param2 AS valid_result" config = { 'query': { @@ -604,17 +479,17 @@ def test_query_with_parameters(self): # Test that a query that relies on parameters fails # when parameters are not supplied via configuration with pytest.raises(ValueError): - gbq.read_gbq(sql_statement, project_id=_get_project_id(), + gbq.read_gbq(sql_statement, project_id=project_id, private_key=self.credentials) # Test that the query is successful because we have supplied # the correct query parameters via the 'config' option - df = gbq.read_gbq(sql_statement, project_id=_get_project_id(), + df = gbq.read_gbq(sql_statement, project_id=project_id, private_key=self.credentials, configuration=config) tm.assert_frame_equal(df, DataFrame({'valid_result': [3]})) - def test_query_inside_configuration(self): + def test_query_inside_configuration(self, project_id): query_no_use = 'SELECT "PI_WRONG" AS valid_string' query = 'SELECT "PI" AS valid_string' config = { @@ -626,26 +501,26 @@ def test_query_inside_configuration(self): # Test that it can't pass query both # inside config and as parameter with pytest.raises(ValueError): - gbq.read_gbq(query_no_use, project_id=_get_project_id(), + gbq.read_gbq(query_no_use, project_id=project_id, private_key=self.credentials, configuration=config) - df = gbq.read_gbq(None, project_id=_get_project_id(), + df = gbq.read_gbq(None, project_id=project_id, private_key=self.credentials, configuration=config) tm.assert_frame_equal(df, DataFrame({'valid_string': ['PI']})) - def test_configuration_without_query(self): + def test_configuration_without_query(self, project_id): sql_statement = 'SELECT 1' config = { 'copy': { "sourceTable": { - "projectId": _get_project_id(), + "projectId": project_id, "datasetId": "publicdata:samples", "tableId": "wikipedia" }, "destinationTable": { - "projectId": _get_project_id(), + "projectId": project_id, "datasetId": "publicdata:samples", "tableId": "wikipedia_copied" }, @@ -654,11 +529,12 @@ def test_configuration_without_query(self): # Test that only 'query' configurations are supported # nor 'copy','load','extract' with pytest.raises(ValueError): - gbq.read_gbq(sql_statement, project_id=_get_project_id(), + gbq.read_gbq(sql_statement, project_id=project_id, private_key=self.credentials, configuration=config) - def test_configuration_raises_value_error_with_multiple_config(self): + def test_configuration_raises_value_error_with_multiple_config( + self, project_id): sql_statement = 'SELECT 1' config = { 'query': { @@ -672,11 +548,11 @@ def test_configuration_raises_value_error_with_multiple_config(self): } # Test that only ValueError is raised with multiple configurations with pytest.raises(ValueError): - gbq.read_gbq(sql_statement, project_id=_get_project_id(), + gbq.read_gbq(sql_statement, project_id=project_id, private_key=self.credentials, configuration=config) - def test_timeout_configuration(self): + def test_timeout_configuration(self, project_id): sql_statement = 'SELECT 1' config = { 'query': { @@ -685,7 +561,7 @@ def test_timeout_configuration(self): } # Test that QueryTimeout error raises with pytest.raises(gbq.QueryTimeout): - gbq.read_gbq(sql_statement, project_id=_get_project_id(), + gbq.read_gbq(sql_statement, project_id=project_id, private_key=self.credentials, configuration=config) @@ -704,25 +580,25 @@ def test_query_response_bytes(self): assert self.gbq_connector.sizeof_fmt(1.208926E24) == "1.0 YB" assert self.gbq_connector.sizeof_fmt(1.208926E28) == "10000.0 YB" - def test_struct(self): + def test_struct(self, project_id): query = """SELECT 1 int_field, STRUCT("a" as letter, 1 as num) struct_field""" - df = gbq.read_gbq(query, project_id=_get_project_id(), + df = gbq.read_gbq(query, project_id=project_id, private_key=self.credentials, dialect='standard') expected = DataFrame([[1, {"letter": "a", "num": 1}]], columns=["int_field", "struct_field"]) tm.assert_frame_equal(df, expected) - def test_array(self): + def test_array(self, project_id): query = """select ["a","x","b","y","c","z"] as letters""" - df = gbq.read_gbq(query, project_id=_get_project_id(), + df = gbq.read_gbq(query, project_id=project_id, private_key=self.credentials, dialect='standard') tm.assert_frame_equal(df, DataFrame([[["a", "x", "b", "y", "c", "z"]]], columns=["letters"])) - def test_array_length_zero(self): + def test_array_length_zero(self, project_id): query = """WITH t as ( SELECT "a" letter, [""] as array_field UNION ALL @@ -731,14 +607,14 @@ def test_array_length_zero(self): select letter, array_field, array_length(array_field) len from t order by letter ASC""" - df = gbq.read_gbq(query, project_id=_get_project_id(), + df = gbq.read_gbq(query, project_id=project_id, private_key=self.credentials, dialect='standard') expected = DataFrame([["a", [""], 1], ["b", [], 0]], columns=["letter", "array_field", "len"]) tm.assert_frame_equal(df, expected) - def test_array_agg(self): + def test_array_agg(self, project_id): query = """WITH t as ( SELECT "a" letter, 1 num UNION ALL @@ -750,16 +626,16 @@ def test_array_agg(self): from t group by letter order by letter ASC""" - df = gbq.read_gbq(query, project_id=_get_project_id(), + df = gbq.read_gbq(query, project_id=project_id, private_key=self.credentials, dialect='standard') tm.assert_frame_equal(df, DataFrame([["a", [1, 3]], ["b", [2]]], columns=["letter", "numbers"])) - def test_array_of_floats(self): + def test_array_of_floats(self, private_key_path, project_id): query = """select [1.1, 2.2, 3.3] as a, 4 as b""" - df = gbq.read_gbq(query, project_id=_get_project_id(), - private_key=_get_private_key_path(), + df = gbq.read_gbq(query, project_id=project_id, + private_key=private_key_path, dialect='standard') tm.assert_frame_equal(df, DataFrame([[[1.1, 2.2, 3.3], 4]], columns=["a", "b"])) @@ -779,7 +655,7 @@ def setup(self, project, credentials): # executed. self.dataset_prefix = _get_dataset_prefix_random() - clean_gbq_environment(self.dataset_prefix, credentials) + clean_gbq_environment(self.dataset_prefix, credentials, project) self.dataset = gbq._Dataset(project, private_key=credentials) self.table = gbq._Table(project, self.dataset_prefix + "1", @@ -791,23 +667,23 @@ def setup(self, project, credentials): self.dataset.create(self.dataset_prefix + "1") self.credentials = credentials yield - clean_gbq_environment(self.dataset_prefix, self.credentials) + clean_gbq_environment(self.dataset_prefix, self.credentials, project) - def test_upload_data(self): + def test_upload_data(self, project_id): test_id = "1" test_size = 20001 df = make_mixed_dataframe_v2(test_size) - gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), + gbq.to_gbq(df, self.destination_table + test_id, project_id, chunksize=10000, private_key=self.credentials) result = gbq.read_gbq("SELECT COUNT(*) AS num_rows FROM {0}" .format(self.destination_table + test_id), - project_id=_get_project_id(), + project_id=project_id, private_key=self.credentials) assert result['num_rows'][0] == test_size - def test_upload_data_if_table_exists_fail(self): + def test_upload_data_if_table_exists_fail(self, project_id): test_id = "2" test_size = 10 df = make_mixed_dataframe_v2(test_size) @@ -815,41 +691,41 @@ def test_upload_data_if_table_exists_fail(self): # Test the default value of if_exists is 'fail' with pytest.raises(gbq.TableCreationError): - gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), + gbq.to_gbq(df, self.destination_table + test_id, project_id, private_key=self.credentials) # Test the if_exists parameter with value 'fail' with pytest.raises(gbq.TableCreationError): - gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), + gbq.to_gbq(df, self.destination_table + test_id, project_id, if_exists='fail', private_key=self.credentials) - def test_upload_data_if_table_exists_append(self): + def test_upload_data_if_table_exists_append(self, project_id): test_id = "3" test_size = 10 df = make_mixed_dataframe_v2(test_size) df_different_schema = tm.makeMixedDataFrame() # Initialize table with sample data - gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), + gbq.to_gbq(df, self.destination_table + test_id, project_id, chunksize=10000, private_key=self.credentials) # Test the if_exists parameter with value 'append' - gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), + gbq.to_gbq(df, self.destination_table + test_id, project_id, if_exists='append', private_key=self.credentials) result = gbq.read_gbq("SELECT COUNT(*) AS num_rows FROM {0}" .format(self.destination_table + test_id), - project_id=_get_project_id(), + project_id=project_id, private_key=self.credentials) assert result['num_rows'][0] == test_size * 2 # Try inserting with a different schema, confirm failure with pytest.raises(gbq.InvalidSchema): gbq.to_gbq(df_different_schema, self.destination_table + test_id, - _get_project_id(), if_exists='append', + project_id, if_exists='append', private_key=self.credentials) - def test_upload_subset_columns_if_table_exists_append(self): + def test_upload_subset_columns_if_table_exists_append(self, project_id): # Issue 24: Upload is succesful if dataframe has columns # which are a subset of the current schema test_id = "16" @@ -858,52 +734,52 @@ def test_upload_subset_columns_if_table_exists_append(self): df_subset_cols = df.iloc[:, :2] # Initialize table with sample data - gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), + gbq.to_gbq(df, self.destination_table + test_id, project_id, chunksize=10000, private_key=self.credentials) # Test the if_exists parameter with value 'append' gbq.to_gbq(df_subset_cols, - self.destination_table + test_id, _get_project_id(), + self.destination_table + test_id, project_id, if_exists='append', private_key=self.credentials) result = gbq.read_gbq("SELECT COUNT(*) AS num_rows FROM {0}" .format(self.destination_table + test_id), - project_id=_get_project_id(), + project_id=project_id, private_key=self.credentials) assert result['num_rows'][0] == test_size * 2 - def test_upload_data_if_table_exists_replace(self): + def test_upload_data_if_table_exists_replace(self, project_id): test_id = "4" test_size = 10 df = make_mixed_dataframe_v2(test_size) df_different_schema = tm.makeMixedDataFrame() # Initialize table with sample data - gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), + gbq.to_gbq(df, self.destination_table + test_id, project_id, chunksize=10000, private_key=self.credentials) # Test the if_exists parameter with the value 'replace'. gbq.to_gbq(df_different_schema, self.destination_table + test_id, - _get_project_id(), if_exists='replace', + project_id, if_exists='replace', private_key=self.credentials) result = gbq.read_gbq("SELECT COUNT(*) AS num_rows FROM {0}" .format(self.destination_table + test_id), - project_id=_get_project_id(), + project_id=project_id, private_key=self.credentials) assert result['num_rows'][0] == 5 - def test_upload_data_if_table_exists_raises_value_error(self): + def test_upload_data_if_table_exists_raises_value_error(self, project_id): test_id = "4" test_size = 10 df = make_mixed_dataframe_v2(test_size) # Test invalid value for if_exists parameter raises value error with pytest.raises(ValueError): - gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), + gbq.to_gbq(df, self.destination_table + test_id, project_id, if_exists='xxxxx', private_key=self.credentials) - def test_google_upload_errors_should_raise_exception(self): + def test_google_upload_errors_should_raise_exception(self, project_id): raise pytest.skip("buggy test") test_id = "5" @@ -915,9 +791,9 @@ def test_google_upload_errors_should_raise_exception(self): with pytest.raises(gbq.StreamingInsertError): gbq.to_gbq(bad_df, self.destination_table + test_id, - _get_project_id(), private_key=self.credentials) + project_id, private_key=self.credentials) - def test_upload_chinese_unicode_data(self): + def test_upload_chinese_unicode_data(self, project_id): test_id = "2" test_size = 6 df = DataFrame(np.random.randn(6, 4), index=range(6), @@ -926,13 +802,13 @@ def test_upload_chinese_unicode_data(self): gbq.to_gbq( df, self.destination_table + test_id, - _get_project_id(), + project_id, private_key=self.credentials, chunksize=10000) result_df = gbq.read_gbq( "SELECT * FROM {0}".format(self.destination_table + test_id), - project_id=_get_project_id(), + project_id=project_id, private_key=self.credentials) assert len(result_df) == test_size @@ -945,7 +821,7 @@ def test_upload_chinese_unicode_data(self): tm.assert_numpy_array_equal(expected.values, result.values) - def test_upload_other_unicode_data(self): + def test_upload_other_unicode_data(self, project_id): test_id = "3" test_size = 3 df = DataFrame({ @@ -959,13 +835,13 @@ def test_upload_other_unicode_data(self): gbq.to_gbq( df, self.destination_table + test_id, - _get_project_id(), + project_id=project_id, private_key=self.credentials, chunksize=10000) result_df = gbq.read_gbq("SELECT * FROM {0}".format( self.destination_table + test_id), - project_id=_get_project_id(), + project_id=project_id, private_key=self.credentials) assert len(result_df) == test_size @@ -978,7 +854,7 @@ def test_upload_other_unicode_data(self): tm.assert_numpy_array_equal(expected.values, result.values) - def test_upload_mixed_float_and_int(self): + def test_upload_mixed_float_and_int(self, project_id): """Test that we can upload a dataframe containing an int64 and float64 column. See: https://github.com/pydata/pandas-gbq/issues/116 """ @@ -991,16 +867,17 @@ def test_upload_mixed_float_and_int(self): gbq.to_gbq( df, self.destination_table + test_id, - _get_project_id(), + project_id=project_id, private_key=self.credentials) result_df = gbq.read_gbq("SELECT * FROM {0}".format( self.destination_table + test_id), - project_id=_get_project_id(), + project_id=project_id, private_key=self.credentials) assert len(result_df) == test_size + # TODO: move generate schema test to unit tests. def test_generate_schema(self): df = tm.makeMixedDataFrame() schema = gbq._generate_bq_schema(df) @@ -1099,19 +976,19 @@ def test_verify_schema_fails_different_structure(self): assert not self.sut.verify_schema( self.dataset_prefix + "1", TABLE_ID + test_id, test_schema_2) - def test_upload_data_flexible_column_order(self): + def test_upload_data_flexible_column_order(self, project_id): test_id = "13" test_size = 10 df = make_mixed_dataframe_v2(test_size) # Initialize table with sample data - gbq.to_gbq(df, self.destination_table + test_id, _get_project_id(), + gbq.to_gbq(df, self.destination_table + test_id, project_id, chunksize=10000, private_key=self.credentials) df_columns_reversed = df[df.columns[::-1]] gbq.to_gbq(df_columns_reversed, self.destination_table + test_id, - _get_project_id(), if_exists='append', + project_id, if_exists='append', private_key=self.credentials) def test_verify_schema_ignores_field_mode(self): @@ -1231,7 +1108,7 @@ def test_schema_is_subset_fails_if_not_subset(self): assert self.sut.schema_is_subset( dataset, table_name, tested_schema) is False - def test_upload_data_with_valid_user_schema(self): + def test_upload_data_with_valid_user_schema(self, project_id): # Issue #46; tests test scenarios with user-provided # schemas df = tm.makeMixedDataFrame() @@ -1241,14 +1118,15 @@ def test_upload_data_with_valid_user_schema(self): {'name': 'C', 'type': 'STRING'}, {'name': 'D', 'type': 'TIMESTAMP'}] destination_table = self.destination_table + test_id - gbq.to_gbq(df, destination_table, _get_project_id(), + gbq.to_gbq(df, destination_table, project_id, private_key=self.credentials, table_schema=test_schema) dataset, table = destination_table.split('.') assert self.table.verify_schema(dataset, table, dict(fields=test_schema)) - def test_upload_data_with_invalid_user_schema_raises_error(self): + def test_upload_data_with_invalid_user_schema_raises_error( + self, project_id): df = tm.makeMixedDataFrame() test_id = "19" test_schema = [{'name': 'A', 'type': 'FLOAT'}, @@ -1257,11 +1135,12 @@ def test_upload_data_with_invalid_user_schema_raises_error(self): {'name': 'D', 'type': 'FLOAT'}] destination_table = self.destination_table + test_id with pytest.raises(gbq.GenericGBQException): - gbq.to_gbq(df, destination_table, _get_project_id(), + gbq.to_gbq(df, destination_table, project_id, private_key=self.credentials, table_schema=test_schema) - def test_upload_data_with_missing_schema_fields_raises_error(self): + def test_upload_data_with_missing_schema_fields_raises_error( + self, project_id): df = tm.makeMixedDataFrame() test_id = "20" test_schema = [{'name': 'A', 'type': 'FLOAT'}, @@ -1269,11 +1148,11 @@ def test_upload_data_with_missing_schema_fields_raises_error(self): {'name': 'C', 'type': 'FLOAT'}] destination_table = self.destination_table + test_id with pytest.raises(gbq.GenericGBQException): - gbq.to_gbq(df, destination_table, _get_project_id(), + gbq.to_gbq(df, destination_table, project_id, private_key=self.credentials, table_schema=test_schema) - def test_upload_data_with_timestamp(self): + def test_upload_data_with_timestamp(self, project_id): test_id = "21" test_size = 6 df = DataFrame(np.random.randn(test_size, 4), index=range(test_size), @@ -1282,12 +1161,12 @@ def test_upload_data_with_timestamp(self): gbq.to_gbq( df, self.destination_table + test_id, - _get_project_id(), + project_id=project_id, private_key=self.credentials) result_df = gbq.read_gbq("SELECT * FROM {0}".format( self.destination_table + test_id), - project_id=_get_project_id(), + project_id=project_id, private_key=self.credentials) assert len(result_df) == test_size @@ -1296,7 +1175,7 @@ def test_upload_data_with_timestamp(self): result = result_df['times'].sort_values() tm.assert_numpy_array_equal(expected.values, result.values) - def test_upload_data_with_different_df_and_user_schema(self): + def test_upload_data_with_different_df_and_user_schema(self, project_id): df = tm.makeMixedDataFrame() df['A'] = df['A'].astype(str) df['B'] = df['B'].astype(str) @@ -1306,7 +1185,7 @@ def test_upload_data_with_different_df_and_user_schema(self): {'name': 'C', 'type': 'STRING'}, {'name': 'D', 'type': 'TIMESTAMP'}] destination_table = self.destination_table + test_id - gbq.to_gbq(df, destination_table, _get_project_id(), + gbq.to_gbq(df, destination_table, project_id, private_key=self.credentials, table_schema=test_schema) dataset, table = destination_table.split('.') @@ -1317,10 +1196,10 @@ def test_list_dataset(self): dataset_id = self.dataset_prefix + "1" assert dataset_id in self.dataset.datasets() - def test_list_table_zero_results(self): + def test_list_table_zero_results(self, project_id): dataset_id = self.dataset_prefix + "2" self.dataset.create(dataset_id) - table_list = gbq._Dataset(_get_project_id(), + table_list = gbq._Dataset(project_id, private_key=self.credentials ).tables(dataset_id) assert len(table_list) == 0 @@ -1352,10 +1231,10 @@ def test_dataset_exists(self): self.dataset.create(dataset_id) assert self.dataset.exists(dataset_id) - def create_table_data_dataset_does_not_exist(self): + def create_table_data_dataset_does_not_exist(self, project_id): dataset_id = self.dataset_prefix + "6" table_id = TABLE_ID + "1" - table_with_new_dataset = gbq._Table(_get_project_id(), dataset_id) + table_with_new_dataset = gbq._Table(project_id, dataset_id) df = make_mixed_dataframe_v2(10) table_with_new_dataset.create(table_id, gbq._generate_bq_schema(df)) assert self.dataset.exists(dataset_id) diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index 85e4f427af73..7b85a562e086 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -5,6 +5,7 @@ from pandas import DataFrame from pandas.compat.numpy import np_datetime64_compat +import pandas_gbq.exceptions from pandas_gbq import gbq try: @@ -39,15 +40,33 @@ def mock_bigquery_client(monkeypatch): gbq.GbqConnector, 'get_client', lambda _: mock_client) -@pytest.fixture(autouse=True) -def no_auth(monkeypatch): +def mock_none_credentials(*args, **kwargs): + return None, None + + +def mock_get_credentials(*args, **kwargs): + import google.auth.credentials + mock_credentials = mock.create_autospec( + google.auth.credentials.Credentials) + return mock_credentials, 'default-project' + + +def mock_get_user_credentials(*args, **kwargs): import google.auth.credentials mock_credentials = mock.create_autospec( google.auth.credentials.Credentials) + return mock_credentials + + +@pytest.fixture(autouse=True) +def no_auth(monkeypatch): + from pandas_gbq import auth + monkeypatch.setattr( + auth, 'get_application_default_credentials', mock_get_credentials) + monkeypatch.setattr( + auth, 'get_user_account_credentials', mock_get_user_credentials) monkeypatch.setattr( - gbq.GbqConnector, - 'get_application_default_credentials', - lambda _: (mock_credentials, 'default-project')) + auth, '_try_credentials', lambda project_id, credentials: credentials) def test_should_return_credentials_path_set_by_env_var(): @@ -81,12 +100,13 @@ def test_to_gbq_should_fail_if_invalid_table_name_passed(): def test_to_gbq_with_no_project_id_given_should_fail(monkeypatch): + from pandas_gbq import auth monkeypatch.setattr( - gbq.GbqConnector, - 'get_application_default_credentials', - lambda _: None) - with pytest.raises(TypeError): + auth, 'get_application_default_credentials', mock_none_credentials) + + with pytest.raises(ValueError) as exception: gbq.to_gbq(DataFrame([[1]]), 'dataset.tablename') + assert 'Could not determine project ID' in str(exception) def test_to_gbq_with_verbose_new_pandas_warns_deprecation(): @@ -163,12 +183,13 @@ def test_to_gbq_with_verbose_old_pandas_no_warnings(recwarn): def test_read_gbq_with_no_project_id_given_should_fail(monkeypatch): + from pandas_gbq import auth monkeypatch.setattr( - gbq.GbqConnector, - 'get_application_default_credentials', - lambda _: None) - with pytest.raises(TypeError): + auth, 'get_application_default_credentials', mock_none_credentials) + + with pytest.raises(ValueError) as exception: gbq.read_gbq('SELECT 1') + assert 'Could not determine project ID' in str(exception) def test_read_gbq_with_inferred_project_id(monkeypatch): @@ -190,17 +211,17 @@ def test_that_parse_data_works_properly(): def test_read_gbq_with_invalid_private_key_json_should_fail(): - with pytest.raises(gbq.InvalidPrivateKeyFormat): + with pytest.raises(pandas_gbq.exceptions.InvalidPrivateKeyFormat): gbq.read_gbq('SELECT 1', project_id='x', private_key='y') def test_read_gbq_with_empty_private_key_json_should_fail(): - with pytest.raises(gbq.InvalidPrivateKeyFormat): + with pytest.raises(pandas_gbq.exceptions.InvalidPrivateKeyFormat): gbq.read_gbq('SELECT 1', project_id='x', private_key='{}') def test_read_gbq_with_private_key_json_wrong_types_should_fail(): - with pytest.raises(gbq.InvalidPrivateKeyFormat): + with pytest.raises(pandas_gbq.exceptions.InvalidPrivateKeyFormat): gbq.read_gbq( 'SELECT 1', project_id='x', private_key='{ "client_email" : 1, "private_key" : True }') @@ -208,13 +229,13 @@ def test_read_gbq_with_private_key_json_wrong_types_should_fail(): def test_read_gbq_with_empty_private_key_file_should_fail(): with tm.ensure_clean() as empty_file_path: - with pytest.raises(gbq.InvalidPrivateKeyFormat): + with pytest.raises(pandas_gbq.exceptions.InvalidPrivateKeyFormat): gbq.read_gbq('SELECT 1', project_id='x', private_key=empty_file_path) def test_read_gbq_with_corrupted_private_key_json_should_fail(): - with pytest.raises(gbq.InvalidPrivateKeyFormat): + with pytest.raises(pandas_gbq.exceptions.InvalidPrivateKeyFormat): gbq.read_gbq( 'SELECT 1', project_id='x', private_key='99999999999999999') @@ -265,3 +286,10 @@ def test_read_gbq_with_verbose_old_pandas_no_warnings(recwarn): mock_version.side_effect = [min_bq_version, pandas_version] gbq.read_gbq('SELECT 1', project_id='my-project', verbose=True) assert len(recwarn) == 0 + + +def test_generate_bq_schema_deprecated(): + # 11121 Deprecation of generate_bq_schema + with pytest.warns(FutureWarning): + df = DataFrame([[1, 'two'], [3, 'four']]) + gbq.generate_bq_schema(df) From c1918c4b7ba63ccd714a57ac7b8978e37c9a5736 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 25 May 2018 16:43:42 -0700 Subject: [PATCH 131/519] DOC: Add how-to guide for authentication (#183) * Clarifies order of authentication methods. * Updates links to external resources. --- packages/pandas-gbq/docs/source/conf.py | 8 ++- .../docs/source/howto/authentication.rst | 62 +++++++++++++++++++ packages/pandas-gbq/docs/source/index.rst | 1 + packages/pandas-gbq/docs/source/intro.rst | 39 ------------ packages/pandas-gbq/pandas_gbq/gbq.py | 46 ++++---------- 5 files changed, 81 insertions(+), 75 deletions(-) create mode 100644 packages/pandas-gbq/docs/source/howto/authentication.rst diff --git a/packages/pandas-gbq/docs/source/conf.py b/packages/pandas-gbq/docs/source/conf.py index 3dfd9f86e94c..3ad2767fa468 100644 --- a/packages/pandas-gbq/docs/source/conf.py +++ b/packages/pandas-gbq/docs/source/conf.py @@ -359,8 +359,12 @@ # texinfo_no_detailmenu = False -# Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'https://docs.python.org/': None} +# Configuration for intersphinx: +intersphinx_mapping = { + 'https://docs.python.org/': None, + 'https://pandas.pydata.org/pandas-docs/stable/': None, + 'https://google-auth.readthedocs.io/en/latest/': None, +} extlinks = {'issue': ('https://github.com/pydata/pandas-gbq/issues/%s', 'GH#'), diff --git a/packages/pandas-gbq/docs/source/howto/authentication.rst b/packages/pandas-gbq/docs/source/howto/authentication.rst new file mode 100644 index 000000000000..9f69ed129842 --- /dev/null +++ b/packages/pandas-gbq/docs/source/howto/authentication.rst @@ -0,0 +1,62 @@ +Authentication +============== + +pandas-gbq `authenticates with the Google BigQuery service +`_ via OAuth 2.0. + +.. _authentication: + + +Authentication with a Service Account +-------------------------------------- + +Using service account credentials is particularly useful when working on +remote servers without access to user input. + +Create a service account key via the `service account key creation page +`_ in +the Google Cloud Platform Console. Select the JSON key type and download the +key file. + +To use service account credentials, set the ``private_key`` parameter to one +of: + +* A file path to the JSON file. +* A string containing the JSON file contents. + +See the `Getting started with authentication on Google Cloud Platform +`_ guide for +more information on service accounts. + +Default Authentication Methods +------------------------------ + +If the ``private_key`` parameter is ``None``, pandas-gbq tries the following +authentication methods: + +1. Application Default Credentials via the :func:`google.auth.default` + function. + + .. note:: + + If pandas-gbq can obtain default credentials but those credentials + cannot be used to query BigQuery, pandas-gbq will also try obtaining + user account credentials. + + A common problem with default credentials when running on Google + Compute Engine is that the VM does not have sufficient scopes to query + BigQuery. + +2. User account credentials. + + pandas-gbq loads cached credentials from a hidden user folder on the + operating system. Override the location of the cached user credentials + by setting the ``PANDAS_GBQ_CREDENTIALS_FILE`` environment variable. + + If pandas-gbq does not find cached credentials, it opens a browser window + asking for you to authenticate to your BigQuery account using the product + name ``pandas GBQ``. + + Additional information on the user credentails authentication mechanism + can be found `here + `__. diff --git a/packages/pandas-gbq/docs/source/index.rst b/packages/pandas-gbq/docs/source/index.rst index 44a61843697c..8e895145a7b4 100644 --- a/packages/pandas-gbq/docs/source/index.rst +++ b/packages/pandas-gbq/docs/source/index.rst @@ -26,6 +26,7 @@ Contents: install.rst intro.rst + howto/authentication.rst reading.rst writing.rst tables.rst diff --git a/packages/pandas-gbq/docs/source/intro.rst b/packages/pandas-gbq/docs/source/intro.rst index 0cad9ae8932f..a7ec65c788ad 100644 --- a/packages/pandas-gbq/docs/source/intro.rst +++ b/packages/pandas-gbq/docs/source/intro.rst @@ -41,42 +41,3 @@ more verbose logs, you can do something like: logger = logging.getLogger('pandas_gbq') logger.setLevel(logging.DEBUG) logger.addHandler(logging.StreamHandler(stream=sys.stdout)) - - -.. _authentication: - -Authentication -'''''''''''''' - -Authentication to the Google ``BigQuery`` service via ``OAuth 2.0`` -is possible with either user or service account credentials. - -Authentication via user account credentials is as simple as following the prompts in a browser window -which will automatically open for you. You authenticate to the specified -``BigQuery`` account using the product name ``pandas GBQ``. -The remote authentication is supported via the ``auth_local_webserver`` in ``read_gbq``. By default, -account credentials are stored in an application-specific hidden user folder on the operating system. You -can override the default credentials location via the ``PANDAS_GBQ_CREDENTIALS_FILE`` environment variable. -Additional information on the authentication mechanism can be found -`here `__. - -Authentication via service account credentials is possible through the `'private_key'` parameter. This method -is particularly useful when working on remote servers (eg. Jupyter Notebooks on remote host). -Additional information on service accounts can be found -`here `__. - -Authentication via ``application default credentials`` is also possible, but only valid -if the parameter ``private_key`` is not provided. This method requires that the -credentials can be fetched from the development environment. Otherwise, the OAuth2 -client-side authentication is used. Additional information can be found on -`application default credentials `__. - -.. note:: - - The `'private_key'` parameter can be set to either the file path of the service account key - in JSON format, or key contents of the service account key in JSON format. - -.. note:: - - A private key can be obtained from the Google developers console by clicking - `here `__. Use JSON key type. diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 0a14c7b19aa8..5fb53bbac712 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -476,23 +476,12 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, The main method a user calls to execute a Query in Google BigQuery and read results into a pandas DataFrame. - The Google Cloud library is used. - Documentation is available `here - `__ + This method uses the Google Cloud client library to make requests to + Google BigQuery, documented `here + `__. - Authentication to the Google BigQuery service is via OAuth 2.0. - - - If "private_key" is not provided: - - By default "application default credentials" are used. - - If default application credentials are not found or are restrictive, - user account credentials are used. In this case, you will be asked to - grant permissions for product name 'pandas GBQ'. - - - If "private_key" is provided: - - Service account credentials will be used to authenticate. + See the :ref:`How to authenticate with Google BigQuery ` + guide for authentication instructions. Parameters ---------- @@ -612,29 +601,18 @@ def to_gbq(dataframe, destination_table, project_id=None, chunksize=None, The main method a user calls to export pandas DataFrame contents to Google BigQuery table. - Google BigQuery API Client Library v2 for Python is used. - Documentation is available `here - `__ - - Authentication to the Google BigQuery service is via OAuth 2.0. - - - If "private_key" is not provided: - - By default "application default credentials" are used. - - If default application credentials are not found or are restrictive, - user account credentials are used. In this case, you will be asked to - grant permissions for product name 'pandas GBQ'. - - - If "private_key" is provided: + This method uses the Google Cloud client library to make requests to + Google BigQuery, documented `here + `__. - Service account credentials will be used to authenticate. + See the :ref:`How to authenticate with Google BigQuery ` + guide for authentication instructions. Parameters ---------- - dataframe : DataFrame + dataframe : pandas.DataFrame DataFrame to be written - destination_table : string + destination_table : str Name of table to be written, in the form 'dataset.tablename' project_id : str (optional when available in environment) Google BigQuery Account project ID. From 24bc389502f87f4d6901e37f28c27e37895658fc Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 25 May 2018 16:56:56 -0700 Subject: [PATCH 132/519] DOC: Add readthedocs YAML The docs build is currently broken. This attempts to fix it by using pip to install pandas-gbq. --- packages/pandas-gbq/.readthedocs.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 packages/pandas-gbq/.readthedocs.yml diff --git a/packages/pandas-gbq/.readthedocs.yml b/packages/pandas-gbq/.readthedocs.yml new file mode 100644 index 000000000000..d0bfff551102 --- /dev/null +++ b/packages/pandas-gbq/.readthedocs.yml @@ -0,0 +1,6 @@ +requirements_file: docs/requirements-docs.txt +build: + image: latest +python: + pip_install: true + version: 3.6 From 8d999059683eb7e28dd39447a991dbb7e447fcda Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Sat, 26 May 2018 08:58:19 -0700 Subject: [PATCH 133/519] TST: Add unit tests for pandas_gbq.auth.get_credentials(). (#184) * TST: Add unit tests for pandas_gbq.auth.get_credentials(). Tests (actual auth mocked out): * Using private key with contents. * Using private key with path. * Using default credentials. * Using cached user credentials. * TST: Use classmethod decorator for mocked methods. See: https://stackoverflow.com/a/29235090/101923 --- packages/pandas-gbq/pandas_gbq/auth.py | 16 +-- packages/pandas-gbq/tests/data/dummy_key.json | 5 + packages/pandas-gbq/tests/unit/test_auth.py | 102 ++++++++++++++++++ 3 files changed, 117 insertions(+), 6 deletions(-) create mode 100644 packages/pandas-gbq/tests/data/dummy_key.json create mode 100644 packages/pandas-gbq/tests/unit/test_auth.py diff --git a/packages/pandas-gbq/pandas_gbq/auth.py b/packages/pandas-gbq/pandas_gbq/auth.py index f401243d09b2..fcf9bc8f3c14 100644 --- a/packages/pandas-gbq/pandas_gbq/auth.py +++ b/packages/pandas-gbq/pandas_gbq/auth.py @@ -39,8 +39,10 @@ def get_service_account_credentials(private_key): import google.auth.transport.requests from google.oauth2.service_account import Credentials + is_path = os.path.isfile(private_key) + try: - if os.path.isfile(private_key): + if is_path: with open(private_key) as f: json_key = json.loads(f.read()) else: @@ -64,11 +66,13 @@ def get_service_account_credentials(private_key): return credentials, json_key.get('project_id') except (KeyError, ValueError, TypeError, AttributeError): raise pandas_gbq.exceptions.InvalidPrivateKeyFormat( - "Private key is missing or invalid. It should be service " - "account private key JSON (file path or string contents) " - "with at least two keys: 'client_email' and 'private_key'. " - "Can be obtained from: https://console.developers.google." - "com/permissions/serviceaccounts") + 'Detected private_key as {}. '.format( + 'path' if is_path else 'contents') + + 'Private key is missing or invalid. It should be service ' + 'account private key JSON (file path or string contents) ' + 'with at least two keys: "client_email" and "private_key". ' + 'Can be obtained from: https://console.developers.google.' + 'com/permissions/serviceaccounts') def get_application_default_credentials(project_id=None): diff --git a/packages/pandas-gbq/tests/data/dummy_key.json b/packages/pandas-gbq/tests/data/dummy_key.json new file mode 100644 index 000000000000..25e17b78c76c --- /dev/null +++ b/packages/pandas-gbq/tests/data/dummy_key.json @@ -0,0 +1,5 @@ + + { + "private_key": "some_key", + "client_email": "service-account@example.com" + } \ No newline at end of file diff --git a/packages/pandas-gbq/tests/unit/test_auth.py b/packages/pandas-gbq/tests/unit/test_auth.py new file mode 100644 index 000000000000..6da3f35d42fe --- /dev/null +++ b/packages/pandas-gbq/tests/unit/test_auth.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- + +import json +import os.path + +try: + import mock +except ImportError: # pragma: NO COVER + from unittest import mock + +from pandas_gbq import auth + + +def test_get_credentials_private_key_contents(monkeypatch): + from google.oauth2 import service_account + + @classmethod + def from_service_account_info(cls, key_info): + mock_credentials = mock.create_autospec(cls) + mock_credentials.with_scopes.return_value = mock_credentials + mock_credentials.refresh.return_value = mock_credentials + return mock_credentials + + monkeypatch.setattr( + service_account.Credentials, + 'from_service_account_info', + from_service_account_info) + private_key = json.dumps({ + 'private_key': 'some_key', + 'client_email': 'service-account@example.com', + 'project_id': 'private-key-project' + }) + credentials, project = auth.get_credentials(private_key=private_key) + + assert credentials is not None + assert project == 'private-key-project' + + +def test_get_credentials_private_key_path(monkeypatch): + from google.oauth2 import service_account + + @classmethod + def from_service_account_info(cls, key_info): + mock_credentials = mock.create_autospec(cls) + mock_credentials.with_scopes.return_value = mock_credentials + mock_credentials.refresh.return_value = mock_credentials + return mock_credentials + + monkeypatch.setattr( + service_account.Credentials, + 'from_service_account_info', + from_service_account_info) + private_key = os.path.join( + os.path.dirname(__file__), '..', 'data', 'dummy_key.json') + credentials, project = auth.get_credentials(private_key=private_key) + + assert credentials is not None + assert project is None + + +def test_get_credentials_default_credentials(monkeypatch): + import google.auth + import google.auth.credentials + import google.cloud.bigquery + + def mock_default_credentials(scopes=None, request=None): + return ( + mock.create_autospec(google.auth.credentials.Credentials), + 'default-project', + ) + + monkeypatch.setattr(google.auth, 'default', mock_default_credentials) + mock_client = mock.create_autospec(google.cloud.bigquery.Client) + monkeypatch.setattr(google.cloud.bigquery, 'Client', mock_client) + + credentials, project = auth.get_credentials() + assert project == 'default-project' + assert credentials is not None + + +def test_get_credentials_load_user_no_default(monkeypatch): + import google.auth + import google.auth.credentials + + def mock_default_credentials(scopes=None, request=None): + return (None, None) + + monkeypatch.setattr(google.auth, 'default', mock_default_credentials) + mock_user_credentials = mock.create_autospec( + google.auth.credentials.Credentials) + + def mock_load_credentials(project_id=None, credentials_path=None): + return mock_user_credentials + + monkeypatch.setattr( + auth, + 'load_user_account_credentials', + mock_load_credentials) + + credentials, project = auth.get_credentials() + assert project is None + assert credentials is mock_user_credentials From 38de54654b785212cd670653baa1dab3334a10c4 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 11 Jun 2018 10:43:51 -0400 Subject: [PATCH 134/519] ENH: Add location parameter to read_gbq and to_gbq (#185) * ENH: Add location parameter to read_gbq and to_gbq This allows queries to be run against datasets in the Tokyo region. Likewise, it enables loading dataframes into Tokyo datasets. The location parameter was added in 0.32.0, so this PR also updates the minimum google-cloud-bigquery version. * DOC: Add location parameter to changelog. Fix test to use private_key parameter so that it passes on Travis. * TST: lock conda google-cloud-bigquery version --- .../pandas-gbq/ci/requirements-3.5-0.18.1.pip | 2 +- packages/pandas-gbq/docs/source/changelog.rst | 2 + packages/pandas-gbq/pandas_gbq/gbq.py | 52 +++++--- packages/pandas-gbq/pandas_gbq/load.py | 6 +- packages/pandas-gbq/pandas_gbq/query.py | 25 ---- packages/pandas-gbq/setup.py | 2 +- packages/pandas-gbq/tests/system/conftest.py | 6 +- packages/pandas-gbq/tests/system/test_gbq.py | 112 +++++++++++------- packages/pandas-gbq/tests/unit/test_gbq.py | 31 +++-- packages/pandas-gbq/tests/unit/test_query.py | 56 --------- 10 files changed, 128 insertions(+), 166 deletions(-) delete mode 100644 packages/pandas-gbq/pandas_gbq/query.py delete mode 100644 packages/pandas-gbq/tests/unit/test_query.py diff --git a/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip b/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip index dd33895c9e57..dc651977af74 100644 --- a/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip +++ b/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip @@ -1,3 +1,3 @@ google-auth==1.4.1 google-auth-oauthlib==0.0.1 -google-cloud-bigquery==0.29.0 +google-cloud-bigquery==0.32.0 diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 7864d81e0e43..3d6fcedd97b5 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -7,6 +7,8 @@ Changelog - Project ID parameter is optional in ``read_gbq`` and ``to_gbq`` when it can inferred from the environment. Note: you must still pass in a project ID when using user-based authentication. (:issue:`103`) +- Add location parameter to ``read_gbq`` and ``to_gbq`` so that pandas-gbq + can work with datasets in the Tokyo region. (:issue:`177`) - Progress bar added for ``to_gbq``, through an optional library `tqdm` as dependency. (:issue:`162`) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 5fb53bbac712..db5c435c29c2 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -34,7 +34,7 @@ def _check_google_client_version(): raise ImportError('Could not import pkg_resources (setuptools).') # https://github.com/GoogleCloudPlatform/google-cloud-python/blob/master/bigquery/CHANGELOG.md - bigquery_minimum_version = pkg_resources.parse_version('0.29.0') + bigquery_minimum_version = pkg_resources.parse_version('0.32.0') BIGQUERY_INSTALLED_VERSION = pkg_resources.get_distribution( 'google-cloud-bigquery').parsed_version @@ -152,12 +152,13 @@ class GbqConnector(object): def __init__(self, project_id, reauth=False, private_key=None, auth_local_webserver=False, - dialect='legacy'): + dialect='legacy', location=None): from google.api_core.exceptions import GoogleAPIError from google.api_core.exceptions import ClientError from pandas_gbq import auth self.http_error = (ClientError, GoogleAPIError) self.project_id = project_id + self.location = location self.reauth = reauth self.private_key = private_key self.auth_local_webserver = auth_local_webserver @@ -215,9 +216,9 @@ def process_http_error(ex): raise GenericGBQException("Reason: {0}".format(ex)) def run_query(self, query, **kwargs): - from google.auth.exceptions import RefreshError from concurrent.futures import TimeoutError - import pandas_gbq.query + from google.auth.exceptions import RefreshError + from google.cloud import bigquery job_config = { 'query': { @@ -243,8 +244,8 @@ def run_query(self, query, **kwargs): logger.info('Requesting query... ') query_reply = self.client.query( query, - job_config=pandas_gbq.query.query_config( - job_config, BIGQUERY_INSTALLED_VERSION)) + job_config=bigquery.QueryJobConfig.from_api_repr(job_config), + location=self.location) logger.info('ok.\nQuery running...') except (RefreshError, ValueError): if self.private_key: @@ -319,7 +320,7 @@ def load_data( try: chunks = load.load_chunks(self.client, dataframe, dataset_id, table_id, chunksize=chunksize, - schema=schema) + schema=schema, location=self.location) if progress_bar and tqdm: chunks = tqdm.tqdm(chunks) for remaining_rows in chunks: @@ -470,7 +471,8 @@ def _parse_data(schema, rows): def read_gbq(query, project_id=None, index_col=None, col_order=None, reauth=False, verbose=None, private_key=None, - auth_local_webserver=False, dialect='legacy', **kwargs): + auth_local_webserver=False, dialect='legacy', location=None, + configuration=None): r"""Load data from Google BigQuery using google-cloud-python The main method a user calls to execute a Query in Google BigQuery @@ -520,10 +522,14 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, compliant with the SQL 2011 standard. For more information see `BigQuery SQL Reference `__ - verbose : None, deprecated - - **kwargs : Arbitrary keyword arguments - configuration (dict): query config parameters for job processing. + location : str (optional) + Location where the query job should run. See the `BigQuery locations + + documentation`__ for a list of available locations. The location must + match that of any datasets used in the query. + .. versionadded:: 0.5.0 + configuration : dict (optional) + Query config parameters for job processing. For example: configuration = {'query': {'useQueryCache': False}} @@ -531,6 +537,8 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, For more information see `BigQuery SQL Reference `__ + verbose : None, deprecated + Returns ------- df: DataFrame @@ -550,9 +558,9 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, raise ValueError("'{0}' is not valid for dialect".format(dialect)) connector = GbqConnector( - project_id, reauth=reauth, private_key=private_key, - dialect=dialect, auth_local_webserver=auth_local_webserver) - schema, rows = connector.run_query(query, **kwargs) + project_id, reauth=reauth, private_key=private_key, dialect=dialect, + auth_local_webserver=auth_local_webserver, location=location) + schema, rows = connector.run_query(query, configuration=configuration) final_df = _parse_data(schema, rows) # Reindex the DataFrame on the provided column @@ -595,7 +603,8 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, def to_gbq(dataframe, destination_table, project_id=None, chunksize=None, verbose=None, reauth=False, if_exists='fail', private_key=None, - auth_local_webserver=False, table_schema=None, progress_bar=True): + auth_local_webserver=False, table_schema=None, location=None, + progress_bar=True): """Write a DataFrame to a Google BigQuery table. The main method a user calls to export pandas DataFrame contents to @@ -648,9 +657,16 @@ def to_gbq(dataframe, destination_table, project_id=None, chunksize=None, of DataFrame columns. See BigQuery API documentation on available names of a field. .. versionadded:: 0.3.1 - verbose : None, deprecated + location : str (optional) + Location where the load job should run. See the `BigQuery locations + + documentation`__ for a list of available locations. The location must + match that of the target dataset. + .. versionadded:: 0.5.0 progress_bar : boolean, True by default. It uses the library `tqdm` to show the progress bar for the upload, chunk by chunk. + .. versionadded:: 0.5.0 + verbose : None, deprecated """ _test_google_api_imports() @@ -670,7 +686,7 @@ def to_gbq(dataframe, destination_table, project_id=None, chunksize=None, connector = GbqConnector( project_id, reauth=reauth, private_key=private_key, - auth_local_webserver=auth_local_webserver) + auth_local_webserver=auth_local_webserver, location=location) dataset_id, table_id = destination_table.rsplit('.', 1) table = _Table(project_id, dataset_id, reauth=reauth, diff --git a/packages/pandas-gbq/pandas_gbq/load.py b/packages/pandas-gbq/pandas_gbq/load.py index 2adb59f6edf6..a7d53f8932f2 100644 --- a/packages/pandas-gbq/pandas_gbq/load.py +++ b/packages/pandas-gbq/pandas_gbq/load.py @@ -44,7 +44,8 @@ def encode_chunks(dataframe, chunksize=None): def load_chunks( - client, dataframe, dataset_id, table_id, chunksize=None, schema=None): + client, dataframe, dataset_id, table_id, chunksize=None, schema=None, + location=None): destination_table = client.dataset(dataset_id).table(table_id) job_config = bigquery.LoadJobConfig() job_config.write_disposition = 'WRITE_APPEND' @@ -71,4 +72,5 @@ def load_chunks( client.load_table_from_file( chunk_buffer, destination_table, - job_config=job_config).result() + job_config=job_config, + location=location).result() diff --git a/packages/pandas-gbq/pandas_gbq/query.py b/packages/pandas-gbq/pandas_gbq/query.py deleted file mode 100644 index 864bbb3749d6..000000000000 --- a/packages/pandas-gbq/pandas_gbq/query.py +++ /dev/null @@ -1,25 +0,0 @@ - -import pkg_resources -from google.cloud import bigquery - - -# Version with query config breaking change. -BIGQUERY_CONFIG_VERSION = pkg_resources.parse_version('0.32.0.dev1') - - -def query_config_old_version(resource): - # Verify that we got a query resource. In newer versions of - # google-cloud-bigquery enough of the configuration is passed on to the - # backend that we can expect a backend validation error instead. - if len(resource) != 1: - raise ValueError("Only one job type must be specified, but " - "given {}".format(','.join(resource.keys()))) - if 'query' not in resource: - raise ValueError("Only 'query' job type is supported") - return bigquery.QueryJobConfig.from_api_repr(resource['query']) - - -def query_config(resource, installed_version): - if installed_version < BIGQUERY_CONFIG_VERSION: - return query_config_old_version(resource) - return bigquery.QueryJobConfig.from_api_repr(resource) diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index 40cfa42765b8..bac3e5febfb6 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -21,7 +21,7 @@ def readme(): 'pandas', 'google-auth', 'google-auth-oauthlib', - 'google-cloud-bigquery>=0.29.0', + 'google-cloud-bigquery>=0.32.0', ] extras = { diff --git a/packages/pandas-gbq/tests/system/conftest.py b/packages/pandas-gbq/tests/system/conftest.py index 18132e9aa316..fdd4d3f6b725 100644 --- a/packages/pandas-gbq/tests/system/conftest.py +++ b/packages/pandas-gbq/tests/system/conftest.py @@ -6,13 +6,13 @@ import pytest -@pytest.fixture +@pytest.fixture(scope='session') def project_id(): return (os.environ.get('GBQ_PROJECT_ID') or os.environ.get('GOOGLE_CLOUD_PROJECT')) # noqa -@pytest.fixture +@pytest.fixture(scope='session') def private_key_path(): path = None if 'TRAVIS_BUILD_DIR' in os.environ: @@ -36,7 +36,7 @@ def private_key_path(): return path -@pytest.fixture +@pytest.fixture(scope='session') def private_key_contents(private_key_path): if private_key_path is None: return None diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index 6cb792afe33c..39b2f4ee306f 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -3,7 +3,6 @@ import sys from datetime import datetime from random import randint -from time import sleep import numpy as np import pandas.util.testing as tm @@ -50,46 +49,46 @@ def gbq_connector(project, credentials): return gbq.GbqConnector(project, private_key=credentials) -def clean_gbq_environment(dataset_prefix, private_key=None, project_id=None): - dataset = gbq._Dataset(project_id, private_key=private_key) - all_datasets = dataset.datasets() - - retry = 3 - while retry > 0: - try: - retry = retry - 1 - for i in range(1, 10): - dataset_id = dataset_prefix + str(i) - if dataset_id in all_datasets: - table = gbq._Table(project_id, dataset_id, - private_key=private_key) - - # Table listing is eventually consistent, so loop until - # all tables no longer appear (max 30 seconds). - table_retry = 30 - all_tables = dataset.tables(dataset_id) - while all_tables and table_retry > 0: - for table_id in all_tables: - try: - table.delete(table_id) - except gbq.NotFoundException: - pass - sleep(1) - table_retry = table_retry - 1 - all_tables = dataset.tables(dataset_id) - - dataset.delete(dataset_id) - retry = 0 - except gbq.GenericGBQException as ex: - # Build in retry logic to work around the following errors : - # An internal error occurred and the request could not be... - # Dataset ... is still in use - error_message = str(ex).lower() - if ('an internal error occurred' in error_message or - 'still in use' in error_message) and retry > 0: - sleep(30) - else: - raise ex +@pytest.fixture(scope='module') +def bigquery_client(project_id, private_key_path): + from google.cloud import bigquery + return bigquery.Client.from_service_account_json( + private_key_path, project=project_id) + + +@pytest.fixture(scope='module') +def tokyo_dataset(bigquery_client): + from google.cloud import bigquery + dataset_id = 'tokyo_{}'.format(_get_dataset_prefix_random()) + dataset_ref = bigquery_client.dataset(dataset_id) + dataset = bigquery.Dataset(dataset_ref) + dataset.location = 'asia-northeast1' + bigquery_client.create_dataset(dataset) + yield dataset_id + bigquery_client.delete_dataset(dataset_ref, delete_contents=True) + + +@pytest.fixture(scope='module') +def tokyo_table(bigquery_client, tokyo_dataset): + table_id = 'tokyo_table' + # Create a random table using DDL. + # https://github.com/GoogleCloudPlatform/golang-samples/blob/2ab2c6b79a1ea3d71d8f91609b57a8fbde07ae5d/bigquery/snippets/snippet.go#L739 + bigquery_client.query( + """CREATE TABLE {}.{} + AS SELECT + 2000 + CAST(18 * RAND() as INT64) as year, + IF(RAND() > 0.5,"foo","bar") as token + FROM UNNEST(GENERATE_ARRAY(0,5,1)) as r + """.format(tokyo_dataset, table_id), + location='asia-northeast1').result() + return table_id + + +def clean_gbq_environment(dataset_prefix, bigquery_client): + for dataset in bigquery_client.list_datasets(): + if not dataset.dataset_id.startswith(dataset_prefix): + continue + bigquery_client.delete_dataset(dataset.reference, delete_contents=True) def make_mixed_dataframe_v2(test_size): @@ -640,6 +639,16 @@ def test_array_of_floats(self, private_key_path, project_id): tm.assert_frame_equal(df, DataFrame([[[1.1, 2.2, 3.3], 4]], columns=["a", "b"])) + def test_tokyo(self, tokyo_dataset, tokyo_table, private_key_path): + df = gbq.read_gbq( + 'SELECT MAX(year) AS max_year FROM {}.{}'.format( + tokyo_dataset, tokyo_table), + dialect='standard', + location='asia-northeast1', + private_key=private_key_path) + print(df) + assert df['max_year'][0] >= 2000 + class TestToGBQIntegration(object): # Changes to BigQuery table schema may take up to 2 minutes as of May 2015 @@ -649,13 +658,13 @@ class TestToGBQIntegration(object): # `__ @pytest.fixture(autouse=True, scope='function') - def setup(self, project, credentials): + def setup(self, project, credentials, bigquery_client): # - PER-TEST FIXTURES - # put here any instruction you want to be run *BEFORE* *EVERY* test is # executed. self.dataset_prefix = _get_dataset_prefix_random() - clean_gbq_environment(self.dataset_prefix, credentials, project) + clean_gbq_environment(self.dataset_prefix, bigquery_client) self.dataset = gbq._Dataset(project, private_key=credentials) self.table = gbq._Table(project, self.dataset_prefix + "1", @@ -667,7 +676,7 @@ def setup(self, project, credentials): self.dataset.create(self.dataset_prefix + "1") self.credentials = credentials yield - clean_gbq_environment(self.dataset_prefix, self.credentials, project) + clean_gbq_environment(self.dataset_prefix, bigquery_client) def test_upload_data(self, project_id): test_id = "1" @@ -1192,6 +1201,21 @@ def test_upload_data_with_different_df_and_user_schema(self, project_id): assert self.table.verify_schema(dataset, table, dict(fields=test_schema)) + def test_upload_data_tokyo( + self, project_id, tokyo_dataset, bigquery_client): + test_size = 10 + df = make_mixed_dataframe_v2(test_size) + tokyo_destination = '{}.to_gbq_test'.format(tokyo_dataset) + + # Initialize table with sample data + gbq.to_gbq( + df, tokyo_destination, project_id, private_key=self.credentials, + location='asia-northeast1') + + table = bigquery_client.get_table( + bigquery_client.dataset(tokyo_dataset).table('to_gbq_test')) + assert table.num_rows > 0 + def test_list_dataset(self): dataset_id = self.dataset_prefix + "1" assert dataset_id in self.dataset.datasets() diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index 7b85a562e086..d3560450f054 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -14,6 +14,12 @@ from unittest import mock +@pytest.fixture +def min_bq_version(): + import pkg_resources + return pkg_resources.parse_version('0.32.0') + + @pytest.fixture(autouse=True) def mock_bigquery_client(monkeypatch): from google.api_core.exceptions import NotFound @@ -109,9 +115,8 @@ def test_to_gbq_with_no_project_id_given_should_fail(monkeypatch): assert 'Could not determine project ID' in str(exception) -def test_to_gbq_with_verbose_new_pandas_warns_deprecation(): +def test_to_gbq_with_verbose_new_pandas_warns_deprecation(min_bq_version): import pkg_resources - min_bq_version = pkg_resources.parse_version('0.29.0') pandas_version = pkg_resources.parse_version('0.23.0') with pytest.warns(FutureWarning), \ mock.patch( @@ -128,9 +133,8 @@ def test_to_gbq_with_verbose_new_pandas_warns_deprecation(): pass -def test_to_gbq_with_not_verbose_new_pandas_warns_deprecation(): +def test_to_gbq_with_not_verbose_new_pandas_warns_deprecation(min_bq_version): import pkg_resources - min_bq_version = pkg_resources.parse_version('0.29.0') pandas_version = pkg_resources.parse_version('0.23.0') with pytest.warns(FutureWarning), \ mock.patch( @@ -147,9 +151,8 @@ def test_to_gbq_with_not_verbose_new_pandas_warns_deprecation(): pass -def test_to_gbq_wo_verbose_w_new_pandas_no_warnings(recwarn): +def test_to_gbq_wo_verbose_w_new_pandas_no_warnings(recwarn, min_bq_version): import pkg_resources - min_bq_version = pkg_resources.parse_version('0.29.0') pandas_version = pkg_resources.parse_version('0.23.0') with mock.patch( 'pkg_resources.Distribution.parsed_version', @@ -163,9 +166,8 @@ def test_to_gbq_wo_verbose_w_new_pandas_no_warnings(recwarn): assert len(recwarn) == 0 -def test_to_gbq_with_verbose_old_pandas_no_warnings(recwarn): +def test_to_gbq_with_verbose_old_pandas_no_warnings(recwarn, min_bq_version): import pkg_resources - min_bq_version = pkg_resources.parse_version('0.29.0') pandas_version = pkg_resources.parse_version('0.22.0') with mock.patch( 'pkg_resources.Distribution.parsed_version', @@ -240,9 +242,8 @@ def test_read_gbq_with_corrupted_private_key_json_should_fail(): 'SELECT 1', project_id='x', private_key='99999999999999999') -def test_read_gbq_with_verbose_new_pandas_warns_deprecation(): +def test_read_gbq_with_verbose_new_pandas_warns_deprecation(min_bq_version): import pkg_resources - min_bq_version = pkg_resources.parse_version('0.29.0') pandas_version = pkg_resources.parse_version('0.23.0') with pytest.warns(FutureWarning), \ mock.patch( @@ -252,9 +253,9 @@ def test_read_gbq_with_verbose_new_pandas_warns_deprecation(): gbq.read_gbq('SELECT 1', project_id='my-project', verbose=True) -def test_read_gbq_with_not_verbose_new_pandas_warns_deprecation(): +def test_read_gbq_with_not_verbose_new_pandas_warns_deprecation( + min_bq_version): import pkg_resources - min_bq_version = pkg_resources.parse_version('0.29.0') pandas_version = pkg_resources.parse_version('0.23.0') with pytest.warns(FutureWarning), \ mock.patch( @@ -264,9 +265,8 @@ def test_read_gbq_with_not_verbose_new_pandas_warns_deprecation(): gbq.read_gbq('SELECT 1', project_id='my-project', verbose=False) -def test_read_gbq_wo_verbose_w_new_pandas_no_warnings(recwarn): +def test_read_gbq_wo_verbose_w_new_pandas_no_warnings(recwarn, min_bq_version): import pkg_resources - min_bq_version = pkg_resources.parse_version('0.29.0') pandas_version = pkg_resources.parse_version('0.23.0') with mock.patch( 'pkg_resources.Distribution.parsed_version', @@ -276,9 +276,8 @@ def test_read_gbq_wo_verbose_w_new_pandas_no_warnings(recwarn): assert len(recwarn) == 0 -def test_read_gbq_with_verbose_old_pandas_no_warnings(recwarn): +def test_read_gbq_with_verbose_old_pandas_no_warnings(recwarn, min_bq_version): import pkg_resources - min_bq_version = pkg_resources.parse_version('0.29.0') pandas_version = pkg_resources.parse_version('0.22.0') with mock.patch( 'pkg_resources.Distribution.parsed_version', diff --git a/packages/pandas-gbq/tests/unit/test_query.py b/packages/pandas-gbq/tests/unit/test_query.py deleted file mode 100644 index 0a89dfb92545..000000000000 --- a/packages/pandas-gbq/tests/unit/test_query.py +++ /dev/null @@ -1,56 +0,0 @@ - -import pkg_resources - -try: - import mock -except ImportError: # pragma: NO COVER - from unittest import mock - -from pandas_gbq import query - - -@mock.patch('google.cloud.bigquery.QueryJobConfig') -def test_query_config_w_old_bq_version(mock_config): - old_version = pkg_resources.parse_version('0.29.0') - query.query_config({'query': {'useLegacySql': False}}, old_version) - mock_config.from_api_repr.assert_called_once_with({'useLegacySql': False}) - - -@mock.patch('google.cloud.bigquery.QueryJobConfig') -def test_query_config_w_dev_bq_version(mock_config): - dev_version = pkg_resources.parse_version('0.32.0.dev1') - query.query_config( - { - 'query': { - 'useLegacySql': False, - }, - 'labels': {'key': 'value'}, - }, - dev_version) - mock_config.from_api_repr.assert_called_once_with( - { - 'query': { - 'useLegacySql': False, - }, - 'labels': {'key': 'value'}, - }) - - -@mock.patch('google.cloud.bigquery.QueryJobConfig') -def test_query_config_w_new_bq_version(mock_config): - dev_version = pkg_resources.parse_version('1.0.0') - query.query_config( - { - 'query': { - 'useLegacySql': False, - }, - 'labels': {'key': 'value'}, - }, - dev_version) - mock_config.from_api_repr.assert_called_once_with( - { - 'query': { - 'useLegacySql': False, - }, - 'labels': {'key': 'value'}, - }) From e19015d91226c0f0145324f6799a857922435980 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 15 Jun 2018 12:54:35 -0400 Subject: [PATCH 135/519] DOC: Prepare for 0.5.0 release (#188) * Update changelog with all changes since 0.4.1. * Address warnings when building the docs due to mis-formed links. * Removes some old references to streaming, which are no longer relevant. --- packages/pandas-gbq/docs/source/changelog.rst | 16 ++++++--- packages/pandas-gbq/docs/source/reading.rst | 14 ++++---- packages/pandas-gbq/docs/source/writing.rst | 34 +++++++------------ packages/pandas-gbq/pandas_gbq/gbq.py | 14 ++++---- 4 files changed, 41 insertions(+), 37 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 3d6fcedd97b5..bc5d243c97fd 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,23 +1,31 @@ Changelog ========= -0.5.0 / TBD ------------ +0.5.0 / 2018-06-15 +------------------ - Project ID parameter is optional in ``read_gbq`` and ``to_gbq`` when it can inferred from the environment. Note: you must still pass in a project ID when using user-based authentication. (:issue:`103`) -- Add location parameter to ``read_gbq`` and ``to_gbq`` so that pandas-gbq - can work with datasets in the Tokyo region. (:issue:`177`) - Progress bar added for ``to_gbq``, through an optional library `tqdm` as dependency. (:issue:`162`) +- Add location parameter to ``read_gbq`` and ``to_gbq`` so that pandas-gbq + can work with datasets in the Tokyo region. (:issue:`177`) + +Documentation +~~~~~~~~~~~~~ +- Add :doc:`authentication how-to guide `. (:issue:`183`) +- Update :doc:`contributing` guide with new paths to tests. (:issue:`154`, + :issue:`164`) Internal changes ~~~~~~~~~~~~~~~~ - Tests now use `nox` to run in multiple Python environments. (:issue:`52`) - Renamed internal modules. (:issue:`154`) +- Refactored auth to an internal auth module. (:issue:`176`) +- Add unit tests for ``get_credentials()``. (:issue:`184`) 0.4.1 / 2018-04-05 ------------------ diff --git a/packages/pandas-gbq/docs/source/reading.rst b/packages/pandas-gbq/docs/source/reading.rst index 16abbdc32dd8..b188c39236d4 100644 --- a/packages/pandas-gbq/docs/source/reading.rst +++ b/packages/pandas-gbq/docs/source/reading.rst @@ -3,8 +3,9 @@ Reading Tables ============== -Suppose you want to load all data from an existing BigQuery table : `test_dataset.test_table` -into a DataFrame using the :func:`~read_gbq` function. +Suppose you want to load all data from an existing BigQuery table +``test_dataset.test_table`` into a DataFrame using the +:func:`~pandas_gbq.read_gbq` function. .. code-block:: python @@ -25,9 +26,9 @@ destination DataFrame as well as a preferred column order as follows: col_order=['col1', 'col2', 'col3'], projectid) -You can specify the query config as parameter to use additional options of your job. -For more information about query configuration parameters see -`here `__. +You can specify the query config as parameter to use additional options of +your job. For more information about query configuration parameters see `here +`__. .. code-block:: python @@ -42,7 +43,8 @@ For more information about query configuration parameters see .. note:: - You can find your project id in the `Google developers console `__. + You can find your project id in the `Google developers console + `__. .. note:: diff --git a/packages/pandas-gbq/docs/source/writing.rst b/packages/pandas-gbq/docs/source/writing.rst index 7a54a3622003..14124d03408f 100644 --- a/packages/pandas-gbq/docs/source/writing.rst +++ b/packages/pandas-gbq/docs/source/writing.rst @@ -3,7 +3,8 @@ Writing DataFrames ================== -Assume we want to write a DataFrame ``df`` into a BigQuery table using :func:`~to_gbq`. +Assume we want to write a DataFrame ``df`` into a BigQuery table using +:func:`~pandas_gbq.to_gbq`. .. ipython:: python @@ -38,21 +39,10 @@ a ``TableCreationError`` if the destination table already exists. .. note:: - If the ``if_exists`` argument is set to ``'append'``, the destination dataframe will - be written to the table using the defined table schema and column types. The - dataframe must contain fields (matching name and type) currently in the destination table. - If the ``if_exists`` argument is set to ``'replace'``, and the existing table has a - different schema, a delay of 2 minutes will be forced to ensure that the new schema - has propagated in the Google environment. See - `Google BigQuery issue 191 `__. - -Writing large DataFrames can result in errors due to size limitations being exceeded. -This can be avoided by setting the ``chunksize`` argument when calling :func:`~to_gbq`. -For example, the following writes ``df`` to a BigQuery table in batches of 10000 rows at a time: - -.. code-block:: python - - to_gbq(df, 'my_dataset.my_table', projectid, chunksize=10000) + If the ``if_exists`` argument is set to ``'append'``, the destination + dataframe will be written to the table using the defined table schema and + column types. The dataframe must contain fields (matching name and type) + currently in the destination table. .. note:: @@ -66,8 +56,10 @@ For example, the following writes ``df`` to a BigQuery table in batches of 10000 .. note:: - While BigQuery uses SQL-like syntax, it has some important differences from traditional - databases both in functionality, API limitations (size and quantity of queries or uploads), - and how Google charges for use of the service. You should refer to `Google BigQuery documentation `__ - often as the service seems to be changing and evolving. BiqQuery is best for analyzing large - sets of data quickly, but it is not a direct replacement for a transactional database. + While BigQuery uses SQL-like syntax, it has some important differences + from traditional databases both in functionality, API limitations (size + and quantity of queries or uploads), and how Google charges for use of the + service. You should refer to `Google BigQuery documentation + `__ often as the service is always + evolving. BiqQuery is best for analyzing large sets of data quickly, but + it is not a direct replacement for a transactional database. diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index db5c435c29c2..ada75839eede 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -524,9 +524,10 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, `__ location : str (optional) Location where the query job should run. See the `BigQuery locations - - documentation`__ for a list of available locations. The location must - match that of any datasets used in the query. + documentation + `__ for a + list of available locations. The location must match that of any + datasets used in the query. .. versionadded:: 0.5.0 configuration : dict (optional) Query config parameters for job processing. @@ -659,9 +660,10 @@ def to_gbq(dataframe, destination_table, project_id=None, chunksize=None, .. versionadded:: 0.3.1 location : str (optional) Location where the load job should run. See the `BigQuery locations - - documentation`__ for a list of available locations. The location must - match that of the target dataset. + documentation + `__ for a + list of available locations. The location must match that of the + target dataset. .. versionadded:: 0.5.0 progress_bar : boolean, True by default. It uses the library `tqdm` to show the progress bar for the upload, chunk by chunk. From dfa98c5e6783737c338f60c039117edb784f81f6 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 25 Jun 2018 11:26:50 -0700 Subject: [PATCH 136/519] DOC: synchronize docs changes with pandas (#190) The docs needs some corrections in order to pass the Pandas docs linter in https://github.com/pandas-dev/pandas/pull/21628 --- packages/pandas-gbq/pandas_gbq/gbq.py | 156 ++++++++++++++------------ 1 file changed, 87 insertions(+), 69 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index ada75839eede..df20e76bb216 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -470,9 +470,9 @@ def _parse_data(schema, rows): def read_gbq(query, project_id=None, index_col=None, col_order=None, - reauth=False, verbose=None, private_key=None, - auth_local_webserver=False, dialect='legacy', location=None, - configuration=None): + reauth=False, private_key=None, auth_local_webserver=False, + dialect='legacy', location=None, configuration=None, + verbose=None): r"""Load data from Google BigQuery using google-cloud-python The main method a user calls to execute a Query in Google BigQuery @@ -488,63 +488,69 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, Parameters ---------- query : str - SQL-Like Query to return data values - project_id : str (optional when available in environment) - Google BigQuery Account project ID. - index_col : str (optional) - Name of result column to use for index in results DataFrame - col_order : list(str) (optional) + SQL-Like Query to return data values. + project_id : str, optional + Google BigQuery Account project ID. Optional when available from + the environment. + index_col : str, optional + Name of result column to use for index in results DataFrame. + col_order : list(str), optional List of BigQuery column names in the desired order for results - DataFrame - reauth : boolean (default False) - Force Google BigQuery to reauthenticate the user. This is useful + DataFrame. + reauth : boolean, default False + Force Google BigQuery to re-authenticate the user. This is useful if multiple accounts are used. - private_key : str (optional) + private_key : str, optional Service account private key in JSON format. Can be file path or string contents. This is useful for remote server - authentication (eg. jupyter iPython notebook on remote host) + authentication (eg. Jupyter/IPython notebook on remote host). auth_local_webserver : boolean, default False - Use the [local webserver flow] instead of the [console flow] when - getting user credentials. A file named bigquery_credentials.dat will - be created in current dir. You can also set PANDAS_GBQ_CREDENTIALS_FILE - environment variable so as to define a specific path to store this - credential (eg. /etc/keys/bigquery.dat). + Use the `local webserver flow`_ instead of the `console flow`_ + when getting user credentials. - .. [local webserver flow] + .. _local webserver flow: http://google-auth-oauthlib.readthedocs.io/en/latest/reference/google_auth_oauthlib.flow.html#google_auth_oauthlib.flow.InstalledAppFlow.run_local_server - .. [console flow] + .. _console flow: http://google-auth-oauthlib.readthedocs.io/en/latest/reference/google_auth_oauthlib.flow.html#google_auth_oauthlib.flow.InstalledAppFlow.run_console - .. versionadded:: 0.2.0 - dialect : {'legacy', 'standard'}, default 'legacy' - 'legacy' : Use BigQuery's legacy SQL dialect. - 'standard' : Use BigQuery's standard SQL (beta), which is - compliant with the SQL 2011 standard. For more information - see `BigQuery SQL Reference - `__ - location : str (optional) + .. versionadded:: 0.2.0 + dialect : str, default 'legacy' + SQL syntax dialect to use. Value can be one of: + + ``'legacy'`` + Use BigQuery's legacy SQL dialect. For more information see + `BigQuery Legacy SQL Reference + `__. + ``'standard'`` + Use BigQuery's standard SQL, which is + compliant with the SQL 2011 standard. For more information + see `BigQuery Standard SQL Reference + `__. + location : str, optional Location where the query job should run. See the `BigQuery locations documentation `__ for a list of available locations. The location must match that of any datasets used in the query. + .. versionadded:: 0.5.0 - configuration : dict (optional) + configuration : dict, optional Query config parameters for job processing. For example: configuration = {'query': {'useQueryCache': False}} - For more information see `BigQuery SQL Reference - `__ - + For more information see `BigQuery REST API Reference + `__. verbose : None, deprecated + Deprecated in Pandas-GBQ 0.4.0. Use the `logging module + to adjust verbosity instead + `__. Returns ------- df: DataFrame - DataFrame representing results of query - + DataFrame representing results of query. """ _test_google_api_imports() @@ -603,9 +609,9 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, def to_gbq(dataframe, destination_table, project_id=None, chunksize=None, - verbose=None, reauth=False, if_exists='fail', private_key=None, + reauth=False, if_exists='fail', private_key=None, auth_local_webserver=False, table_schema=None, location=None, - progress_bar=True): + progress_bar=True, verbose=None): """Write a DataFrame to a Google BigQuery table. The main method a user calls to export pandas DataFrame contents to @@ -621,54 +627,66 @@ def to_gbq(dataframe, destination_table, project_id=None, chunksize=None, Parameters ---------- dataframe : pandas.DataFrame - DataFrame to be written + DataFrame to be written to a Google BigQuery table. destination_table : str - Name of table to be written, in the form 'dataset.tablename' - project_id : str (optional when available in environment) - Google BigQuery Account project ID. - chunksize : int (default None) - Number of rows to be inserted in each chunk from the dataframe. Use - ``None`` to load the dataframe in a single chunk. - reauth : boolean (default False) - Force Google BigQuery to reauthenticate the user. This is useful + Name of table to be written, in the form ``dataset.tablename``. + project_id : str, optional + Google BigQuery Account project ID. Optional when available from + the environment. + chunksize : int, optional + Number of rows to be inserted in each chunk from the dataframe. + Set to ``None`` to load the whole dataframe at once. + reauth : bool, default False + Force Google BigQuery to re-authenticate the user. This is useful if multiple accounts are used. - if_exists : {'fail', 'replace', 'append'}, default 'fail' - 'fail': If table exists, do nothing. - 'replace': If table exists, drop it, recreate it, and insert data. - 'append': If table exists and the dataframe schema is a subset of - the destination table schema, insert data. Create destination table - if does not exist. - private_key : str (optional) + if_exists : str, default 'fail' + Behavior when the destination table exists. Value can be one of: + + ``'fail'`` + If table exists, do nothing. + ``'replace'`` + If table exists, drop it, recreate it, and insert data. + ``'append'`` + If table exists, insert data. Create if does not exist. + private_key : str, optional Service account private key in JSON format. Can be file path or string contents. This is useful for remote server - authentication (eg. jupyter iPython notebook on remote host) - auth_local_webserver : boolean, default False - Use the [local webserver flow] instead of the [console flow] when - getting user credentials. + authentication (eg. Jupyter/IPython notebook on remote host). + auth_local_webserver : bool, default False + Use the `local webserver flow`_ instead of the `console flow`_ + when getting user credentials. - .. [local webserver flow] + .. _local webserver flow: http://google-auth-oauthlib.readthedocs.io/en/latest/reference/google_auth_oauthlib.flow.html#google_auth_oauthlib.flow.InstalledAppFlow.run_local_server - .. [console flow] + .. _console flow: http://google-auth-oauthlib.readthedocs.io/en/latest/reference/google_auth_oauthlib.flow.html#google_auth_oauthlib.flow.InstalledAppFlow.run_console + .. versionadded:: 0.2.0 - table_schema : list of dicts - List of BigQuery table fields to which according DataFrame columns - conform to, e.g. `[{'name': 'col1', 'type': 'STRING'},...]`. If - schema is not provided, it will be generated according to dtypes - of DataFrame columns. See BigQuery API documentation on available - names of a field. + table_schema : list of dicts, optional + List of BigQuery table fields to which according DataFrame + columns conform to, e.g. ``[{'name': 'col1', 'type': + 'STRING'},...]``. If schema is not provided, it will be + generated according to dtypes of DataFrame columns. See + BigQuery API documentation on available names of a field. + .. versionadded:: 0.3.1 - location : str (optional) + location : str, optional Location where the load job should run. See the `BigQuery locations documentation `__ for a list of available locations. The location must match that of the target dataset. + .. versionadded:: 0.5.0 - progress_bar : boolean, True by default. It uses the library `tqdm` to show - the progress bar for the upload, chunk by chunk. + progress_bar : bool, default True + Use the library `tqdm` to show the progress bar for the upload, + chunk by chunk. + .. versionadded:: 0.5.0 - verbose : None, deprecated + verbose : bool, deprecated + Deprecated in Pandas-GBQ 0.4.0. Use the `logging module + to adjust verbosity instead + `__. """ _test_google_api_imports() From 4a3d0303a18b18474eecce5a7eed838472c31623 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 26 Jun 2018 09:22:38 -0700 Subject: [PATCH 137/519] Add anchor links to versions in the changelog (#191) Uses [Sphinx syntax for cross-reference labels](https://stackoverflow.com/a/19543591/101923). --- packages/pandas-gbq/docs/source/changelog.rst | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index bc5d243c97fd..c540319e9b73 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,6 +1,8 @@ Changelog ========= +.. _changelog-0.5.0: + 0.5.0 / 2018-06-15 ------------------ @@ -27,12 +29,16 @@ Internal changes - Refactored auth to an internal auth module. (:issue:`176`) - Add unit tests for ``get_credentials()``. (:issue:`184`) +.. _changelog-0.4.1: + 0.4.1 / 2018-04-05 ------------------ - Only show ``verbose`` deprecation warning if Pandas version does not populate it. (:issue:`157`) +.. _changelog-0.4.0: + 0.4.0 / 2018-04-03 ------------------ @@ -50,6 +56,8 @@ Internal changes Messages use the logging module instead of printing progress directly to standard output. (:issue:`12`) +.. _changelog-0.3.1: + 0.3.1 / 2018-02-13 ------------------ @@ -58,6 +66,8 @@ Internal changes - Fix an issue where a dataframe containing both integer and floating point columns could not be uploaded with ``to_gbq`` (:issue:`116`) - ``to_gbq`` now uses ``to_csv`` to avoid manually looping over rows in a dataframe (should result in faster table uploads) (:issue:`96`) +.. _changelog-0.3.0: + 0.3.0 / 2018-01-03 ------------------ @@ -65,6 +75,8 @@ Internal changes - Structs and arrays are now named properly (:issue:`23`) and BigQuery functions like ``array_agg`` no longer run into errors during type conversion (:issue:`22`). - :func:`to_gbq` now uses a load job instead of the streaming API. Remove ``StreamingInsertError`` class, as it is no longer used by :func:`to_gbq`. (:issue:`7`, :issue:`75`) +.. _changelog-0.2.1: + 0.2.1 / 2017-11-27 ------------------ @@ -72,6 +84,8 @@ Internal changes - Environment variable ``PANDAS_GBQ_CREDENTIALS_FILE`` can now be used to override the default location where the BigQuery user account credentials are stored. (:issue:`86`) - BigQuery user account credentials are now stored in an application-specific hidden user folder on the operating system. (:issue:`41`) +.. _changelog-0.2.0: + 0.2.0 / 2017-07-24 ------------------ @@ -81,21 +95,29 @@ Internal changes - :func:`read_gbq` now has a ``auth_local_webserver`` boolean argument for controlling whether to use web server or console flow when getting user credentials. Replaces `--noauth_local_webserver` command line argument. (:issue:`35`) - :func:`read_gbq` now displays the BigQuery Job ID and standard price in verbose output. (:issue:`70` and :issue:`71`) +.. _changelog-0.1.6: + 0.1.6 / 2017-05-03 ------------------ - All gbq errors will simply be subclasses of ``ValueError`` and no longer inherit from the deprecated ``PandasError``. +.. _changelog-0.1.4: + 0.1.4 / 2017-03-17 ------------------ - ``InvalidIndexColumn`` will be raised instead of ``InvalidColumnOrder`` in :func:`read_gbq` when the index column specified does not exist in the BigQuery schema. (:issue:`6`) +.. _changelog-0.1.3: + 0.1.3 / 2017-03-04 ------------------ - Bug with appending to a BigQuery table where fields have modes (NULLABLE,REQUIRED,REPEATED) specified. These modes were compared versus the remote schema and writing a table via :func:`to_gbq` would previously raise. (:issue:`13`) +.. _changelog-0.1.2: + 0.1.2 / 2017-02-23 ------------------ From 151a53d99d97e431d744f79a51a7df0aebe810ed Mon Sep 17 00:00:00 2001 From: Anthony Delage <1356592+anthonydelage@users.noreply.github.com> Date: Thu, 26 Jul 2018 10:47:06 -0400 Subject: [PATCH 138/519] Use general float format when writing to CSV buffer to prevent numerical overload (#193) * Write to CSV stream with general float format. * Specify number of significant digits for float format. * Change format to '%.15g' and add tests. * Fixing style errors. * Define string as unicode in to_gbq's float test. * Update Changelog. --- packages/pandas-gbq/docs/source/changelog.rst | 9 +++++++++ packages/pandas-gbq/pandas_gbq/load.py | 2 +- packages/pandas-gbq/tests/unit/test_load.py | 15 +++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index c540319e9b73..6600e52cd55d 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,6 +1,15 @@ Changelog ========= +.. _changelog-0.5.1: + +0.5.1 / (Unreleased) +-------------------- + +- Use general float with 15 decimal digit precision when writing to local + CSV buffer in ``to_gbq``. This prevents numerical overflow in certain + edge cases. (:issue:`192`) + .. _changelog-0.5.0: 0.5.0 / 2018-06-15 diff --git a/packages/pandas-gbq/pandas_gbq/load.py b/packages/pandas-gbq/pandas_gbq/load.py index a7d53f8932f2..436eb2d1706b 100644 --- a/packages/pandas-gbq/pandas_gbq/load.py +++ b/packages/pandas-gbq/pandas_gbq/load.py @@ -15,7 +15,7 @@ def encode_chunk(dataframe): csv_buffer = six.StringIO() dataframe.to_csv( csv_buffer, index=False, header=False, encoding='utf-8', - date_format='%Y-%m-%d %H:%M:%S.%f') + float_format='%.15g', date_format='%Y-%m-%d %H:%M:%S.%f') # Convert to a BytesIO buffer so that unicode text is properly handled. # See: https://github.com/pydata/pandas-gbq/issues/106 diff --git a/packages/pandas-gbq/tests/unit/test_load.py b/packages/pandas-gbq/tests/unit/test_load.py index 398a01ae23f2..fdbedc4607fc 100644 --- a/packages/pandas-gbq/tests/unit/test_load.py +++ b/packages/pandas-gbq/tests/unit/test_load.py @@ -4,6 +4,7 @@ import pandas from pandas_gbq import load +from io import StringIO def test_encode_chunk_with_unicode(): @@ -20,6 +21,20 @@ def test_encode_chunk_with_unicode(): assert u'信用卡' in csv_string +def test_encode_chunk_with_floats(): + """Test that floats in a dataframe are encoded with at most 15 significant + figures. + + See: https://github.com/pydata/pandas-gbq/issues/192 + """ + input_csv = StringIO(u'01/01/17 23:00,1.05148,1.05153,1.05148,1.05153,4') + df = pandas.read_csv(input_csv, header=None) + csv_buffer = load.encode_chunk(df) + csv_bytes = csv_buffer.read() + csv_string = csv_bytes.decode('utf-8') + assert '1.05153' in csv_string + + def test_encode_chunks_splits_dataframe(): df = pandas.DataFrame(numpy.random.randn(6, 4), index=range(6)) chunks = list(load.encode_chunks(df, chunksize=2)) From 0dc42838987a5d63b9dfd58e5f3368e6c6b3d5fd Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 15 Aug 2018 12:13:04 -0700 Subject: [PATCH 139/519] CLN: Warn that default dialect is changing to "standard" (#196) * CLN: Warn that default dialect is changing to "standard" Towards issue #195 * Increment stacklevel so line where read_gbq is called is highlighted. * Update pyenv * Try Python 2.7.15 via Travis build instead of pyenv * Add to changelog * Remove unused TRAVIS_PYTHON_VERSION env var. --- packages/pandas-gbq/.travis.yml | 24 +-- packages/pandas-gbq/docs/source/changelog.rst | 7 +- packages/pandas-gbq/pandas_gbq/gbq.py | 13 +- packages/pandas-gbq/tests/system/test_gbq.py | 166 +++++++++++------- packages/pandas-gbq/tests/unit/test_gbq.py | 34 +++- 5 files changed, 154 insertions(+), 90 deletions(-) diff --git a/packages/pandas-gbq/.travis.yml b/packages/pandas-gbq/.travis.yml index cbd7695150c9..b1ab8f9c3d14 100644 --- a/packages/pandas-gbq/.travis.yml +++ b/packages/pandas-gbq/.travis.yml @@ -1,22 +1,26 @@ -sudo: false - language: python +matrix: + include: + - os: linux + python: 2.7 + env: PYTHON=2.7 PANDAS=0.19.2 COVERAGE='false' LINT='true' + - os: linux + python: 3.5 + env: PYTHON=3.5 PANDAS=0.18.1 COVERAGE='true' LINT='false' + - os: linux + python: 3.6 + env: PYTHON=3.6 PANDAS=0.20.1 COVERAGE='false' LINT='false' + - os: linux + python: 3.6 + env: PYTHON=3.6 PANDAS=MASTER COVERAGE='false' LINT='true' env: - - PYTHON=2.7 PANDAS=0.19.2 COVERAGE='false' LINT='true' PYENV_VERSION=2.7.14 - - PYTHON=3.5 PANDAS=0.18.1 COVERAGE='true' LINT='false' PYENV_VERSION=3.5.4 - - PYTHON=3.6 PANDAS=0.20.1 COVERAGE='false' LINT='false' PYENV_VERSION=3.6.1 - - PYTHON=3.6 PANDAS=MASTER COVERAGE='false' LINT='true' PYENV_VERSION=3.6.1 before_install: - echo "before_install" - source ci/travis_process_gbq_encryption.sh install: - # work around https://github.com/travis-ci/travis-ci/issues/8363 - # https://github.com/pre-commit/pre-commit/commit/e3ab8902692e896da9ded42bd4d76ea4e1de359d - - pyenv install -s $PYENV_VERSION - - pyenv global system $PYENV_VERSION # Upgrade setuptools and pip to work around # https://github.com/pypa/setuptools/issues/885 - pip install --upgrade setuptools diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 6600e52cd55d..47a43f9fba7f 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,11 +1,14 @@ Changelog ========= -.. _changelog-0.5.1: +.. _changelog-0.6.0: -0.5.1 / (Unreleased) +0.6.0 / 2018-08-15 -------------------- +- Warn when ``dialect`` is not passed in to ``read_gbq``. The default dialect + will be changing from 'legacy' to 'standard' in a future version. + (:issue:`195`) - Use general float with 15 decimal digit precision when writing to local CSV buffer in ``to_gbq``. This prevents numerical overflow in certain edge cases. (:issue:`192`) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index df20e76bb216..e7642ccd61a5 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -471,7 +471,7 @@ def _parse_data(schema, rows): def read_gbq(query, project_id=None, index_col=None, col_order=None, reauth=False, private_key=None, auth_local_webserver=False, - dialect='legacy', location=None, configuration=None, + dialect=None, location=None, configuration=None, verbose=None): r"""Load data from Google BigQuery using google-cloud-python @@ -515,6 +515,8 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, .. versionadded:: 0.2.0 dialect : str, default 'legacy' + Note: The default value is changing to 'standard' in a future verion. + SQL syntax dialect to use. Value can be one of: ``'legacy'`` @@ -552,6 +554,13 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, df: DataFrame DataFrame representing results of query. """ + if dialect is None: + dialect = 'legacy' + warnings.warn( + 'The default value for dialect is changing to "standard" in a ' + 'future version. Pass in dialect="legacy" to disable this ' + 'warning.', + FutureWarning, stacklevel=2) _test_google_api_imports() @@ -559,7 +568,7 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, warnings.warn( "verbose is deprecated and will be removed in " "a future version. Set logging level in order to vary " - "verbosity", FutureWarning, stacklevel=1) + "verbosity", FutureWarning, stacklevel=2) if dialect not in ('legacy', 'standard'): raise ValueError("'{0}' is not valid for dialect".format(dialect)) diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index 39b2f4ee306f..bb91befb4c0e 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -127,7 +127,8 @@ def test_should_be_able_to_get_results_from_query(self, gbq_connector): def test_should_read(project, credentials): query = 'SELECT "PI" AS valid_string' - df = gbq.read_gbq(query, project_id=project, private_key=credentials) + df = gbq.read_gbq( + query, project_id=project, private_key=credentials, dialect='legacy') tm.assert_frame_equal(df, DataFrame({'valid_string': ['PI']})) @@ -145,25 +146,29 @@ def setup(self, project, credentials): def test_should_properly_handle_valid_strings(self, project_id): query = 'SELECT "PI" AS valid_string' df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') tm.assert_frame_equal(df, DataFrame({'valid_string': ['PI']})) def test_should_properly_handle_empty_strings(self, project_id): query = 'SELECT "" AS empty_string' df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') tm.assert_frame_equal(df, DataFrame({'empty_string': [""]})) def test_should_properly_handle_null_strings(self, project_id): query = 'SELECT STRING(NULL) AS null_string' df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') tm.assert_frame_equal(df, DataFrame({'null_string': [None]})) def test_should_properly_handle_valid_integers(self, project_id): query = 'SELECT INTEGER(3) AS valid_integer' df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') tm.assert_frame_equal(df, DataFrame({'valid_integer': [3]})) def test_should_properly_handle_nullable_integers(self, project_id): @@ -171,14 +176,16 @@ def test_should_properly_handle_nullable_integers(self, project_id): (SELECT 1 AS nullable_integer), (SELECT NULL AS nullable_integer)''' df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') tm.assert_frame_equal( df, DataFrame({'nullable_integer': [1, None]}).astype(object)) def test_should_properly_handle_valid_longs(self, project_id): query = 'SELECT 1 << 62 AS valid_long' df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') tm.assert_frame_equal( df, DataFrame({'valid_long': [1 << 62]})) @@ -187,21 +194,24 @@ def test_should_properly_handle_nullable_longs(self, project_id): (SELECT 1 << 62 AS nullable_long), (SELECT NULL AS nullable_long)''' df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') tm.assert_frame_equal( df, DataFrame({'nullable_long': [1 << 62, None]}).astype(object)) def test_should_properly_handle_null_integers(self, project_id): query = 'SELECT INTEGER(NULL) AS null_integer' df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') tm.assert_frame_equal(df, DataFrame({'null_integer': [None]})) def test_should_properly_handle_valid_floats(self, project_id): from math import pi query = 'SELECT PI() AS valid_float' df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') tm.assert_frame_equal(df, DataFrame( {'valid_float': [pi]})) @@ -211,7 +221,8 @@ def test_should_properly_handle_nullable_floats(self, project_id): (SELECT PI() AS nullable_float), (SELECT NULL AS nullable_float)''' df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') tm.assert_frame_equal( df, DataFrame({'nullable_float': [pi, None]})) @@ -219,7 +230,8 @@ def test_should_properly_handle_valid_doubles(self, project_id): from math import pi query = 'SELECT PI() * POW(10, 307) AS valid_double' df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') tm.assert_frame_equal(df, DataFrame( {'valid_double': [pi * 10 ** 307]})) @@ -229,27 +241,31 @@ def test_should_properly_handle_nullable_doubles(self, project_id): (SELECT PI() * POW(10, 307) AS nullable_double), (SELECT NULL AS nullable_double)''' df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') tm.assert_frame_equal( df, DataFrame({'nullable_double': [pi * 10 ** 307, None]})) def test_should_properly_handle_null_floats(self, project_id): query = 'SELECT FLOAT(NULL) AS null_float' df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') tm.assert_frame_equal(df, DataFrame({'null_float': [np.nan]})) def test_should_properly_handle_timestamp_unix_epoch(self, project_id): query = 'SELECT TIMESTAMP("1970-01-01 00:00:00") AS unix_epoch' df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') tm.assert_frame_equal(df, DataFrame( {'unix_epoch': [np.datetime64('1970-01-01T00:00:00.000000Z')]})) def test_should_properly_handle_arbitrary_timestamp(self, project_id): query = 'SELECT TIMESTAMP("2004-09-15 05:00:00") AS valid_timestamp' df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') tm.assert_frame_equal(df, DataFrame({ 'valid_timestamp': [np.datetime64('2004-09-15T05:00:00.000000Z')] })) @@ -257,25 +273,29 @@ def test_should_properly_handle_arbitrary_timestamp(self, project_id): def test_should_properly_handle_null_timestamp(self, project_id): query = 'SELECT TIMESTAMP(NULL) AS null_timestamp' df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') tm.assert_frame_equal(df, DataFrame({'null_timestamp': [NaT]})) def test_should_properly_handle_true_boolean(self, project_id): query = 'SELECT BOOLEAN(TRUE) AS true_boolean' df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') tm.assert_frame_equal(df, DataFrame({'true_boolean': [True]})) def test_should_properly_handle_false_boolean(self, project_id): query = 'SELECT BOOLEAN(FALSE) AS false_boolean' df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') tm.assert_frame_equal(df, DataFrame({'false_boolean': [False]})) def test_should_properly_handle_null_boolean(self, project_id): query = 'SELECT BOOLEAN(NULL) AS null_boolean' df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') tm.assert_frame_equal(df, DataFrame({'null_boolean': [None]})) def test_should_properly_handle_nullable_booleans(self, project_id): @@ -283,7 +303,8 @@ def test_should_properly_handle_nullable_booleans(self, project_id): (SELECT BOOLEAN(TRUE) AS nullable_boolean), (SELECT NULL AS nullable_boolean)''' df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') tm.assert_frame_equal( df, DataFrame({'nullable_boolean': [True, None]}).astype(object)) @@ -300,14 +321,16 @@ def test_unicode_string_conversion_and_normalization(self, project_id): query = 'SELECT "{0}" AS unicode_string'.format(unicode_string) df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') tm.assert_frame_equal(df, correct_test_datatype) def test_index_column(self, project_id): query = "SELECT 'a' AS string_1, 'b' AS string_2" result_frame = gbq.read_gbq(query, project_id=project_id, index_col="string_1", - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') correct_frame = DataFrame( {'string_1': ['a'], 'string_2': ['b']}).set_index("string_1") assert result_frame.index.name == correct_frame.index.name @@ -317,7 +340,8 @@ def test_column_order(self, project_id): col_order = ['string_3', 'string_1', 'string_2'] result_frame = gbq.read_gbq(query, project_id=project_id, col_order=col_order, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') correct_frame = DataFrame({'string_1': ['a'], 'string_2': [ 'b'], 'string_3': ['c']})[col_order] tm.assert_frame_equal(result_frame, correct_frame) @@ -330,14 +354,16 @@ def test_read_gbq_raises_invalid_column_order(self, project_id): with pytest.raises(gbq.InvalidColumnOrder): gbq.read_gbq(query, project_id=project_id, col_order=col_order, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') def test_column_order_plus_index(self, project_id): query = "SELECT 'a' AS string_1, 'b' AS string_2, 'c' AS string_3" col_order = ['string_3', 'string_2'] result_frame = gbq.read_gbq(query, project_id=project_id, index_col='string_1', col_order=col_order, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') correct_frame = DataFrame( {'string_1': ['a'], 'string_2': ['b'], 'string_3': ['c']}) correct_frame.set_index('string_1', inplace=True) @@ -352,13 +378,15 @@ def test_read_gbq_raises_invalid_index_column(self, project_id): with pytest.raises(gbq.InvalidIndexColumn): gbq.read_gbq(query, project_id=project_id, index_col='string_bbb', col_order=col_order, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') def test_malformed_query(self, project_id): with pytest.raises(gbq.GenericGBQException): gbq.read_gbq("SELCET * FORM [publicdata:samples.shakespeare]", project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') def test_bad_project_id(self): with pytest.raises(gbq.GenericGBQException): @@ -370,7 +398,8 @@ def test_bad_table_name(self, project_id): with pytest.raises(gbq.GenericGBQException): gbq.read_gbq("SELECT * FROM [publicdata:samples.nope]", project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') def test_download_dataset_larger_than_200k_rows(self, project_id): test_size = 200005 @@ -380,7 +409,8 @@ def test_download_dataset_larger_than_200k_rows(self, project_id): "GROUP EACH BY id ORDER BY id ASC LIMIT {0}" .format(test_size), project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') assert len(df.drop_duplicates()) == test_size def test_zero_rows(self, project_id): @@ -390,7 +420,8 @@ def test_zero_rows(self, project_id): "FROM [publicdata:samples.wikipedia] " "WHERE timestamp=-9999999", project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') page_array = np.zeros( (0,), dtype=[('title', object), ('id', np.dtype(int)), ('is_bot', np.dtype(bool)), ('ts', 'M8[ns]')]) @@ -420,10 +451,11 @@ def test_standard_sql(self, project_id): "`publicdata.samples.wikipedia` LIMIT 10" # Test that a standard sql statement fails when using - # the legacy SQL dialect (default value) + # the legacy SQL dialect. with pytest.raises(gbq.GenericGBQException): gbq.read_gbq(standard_sql, project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') # Test that a standard sql statement succeeds when # setting dialect='standard' @@ -432,21 +464,6 @@ def test_standard_sql(self, project_id): private_key=self.credentials) assert len(df.drop_duplicates()) == 10 - def test_invalid_option_for_sql_dialect(self, project_id): - sql_statement = "SELECT DISTINCT id FROM " \ - "`publicdata.samples.wikipedia` LIMIT 10" - - # Test that an invalid option for `dialect` raises ValueError - with pytest.raises(ValueError): - gbq.read_gbq(sql_statement, project_id=project_id, - dialect='invalid', - private_key=self.credentials) - - # Test that a correct option for dialect succeeds - # to make sure ValueError was due to invalid dialect - gbq.read_gbq(sql_statement, project_id=project_id, - dialect='standard', private_key=self.credentials) - def test_query_with_parameters(self, project_id): sql_statement = "SELECT @param1 + @param2 AS valid_result" config = { @@ -479,13 +496,15 @@ def test_query_with_parameters(self, project_id): # when parameters are not supplied via configuration with pytest.raises(ValueError): gbq.read_gbq(sql_statement, project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') # Test that the query is successful because we have supplied # the correct query parameters via the 'config' option df = gbq.read_gbq(sql_statement, project_id=project_id, private_key=self.credentials, - configuration=config) + configuration=config, + dialect='legacy') tm.assert_frame_equal(df, DataFrame({'valid_result': [3]})) def test_query_inside_configuration(self, project_id): @@ -502,11 +521,13 @@ def test_query_inside_configuration(self, project_id): with pytest.raises(ValueError): gbq.read_gbq(query_no_use, project_id=project_id, private_key=self.credentials, - configuration=config) + configuration=config, + dialect='legacy') df = gbq.read_gbq(None, project_id=project_id, private_key=self.credentials, - configuration=config) + configuration=config, + dialect='legacy') tm.assert_frame_equal(df, DataFrame({'valid_string': ['PI']})) def test_configuration_without_query(self, project_id): @@ -530,7 +551,8 @@ def test_configuration_without_query(self, project_id): with pytest.raises(ValueError): gbq.read_gbq(sql_statement, project_id=project_id, private_key=self.credentials, - configuration=config) + configuration=config, + dialect='legacy') def test_configuration_raises_value_error_with_multiple_config( self, project_id): @@ -549,7 +571,8 @@ def test_configuration_raises_value_error_with_multiple_config( with pytest.raises(ValueError): gbq.read_gbq(sql_statement, project_id=project_id, private_key=self.credentials, - configuration=config) + configuration=config, + dialect='legacy') def test_timeout_configuration(self, project_id): sql_statement = 'SELECT 1' @@ -562,7 +585,8 @@ def test_timeout_configuration(self, project_id): with pytest.raises(gbq.QueryTimeout): gbq.read_gbq(sql_statement, project_id=project_id, private_key=self.credentials, - configuration=config) + configuration=config, + dialect='legacy') def test_query_response_bytes(self): assert self.gbq_connector.sizeof_fmt(999) == "999.0 B" @@ -689,7 +713,8 @@ def test_upload_data(self, project_id): result = gbq.read_gbq("SELECT COUNT(*) AS num_rows FROM {0}" .format(self.destination_table + test_id), project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') assert result['num_rows'][0] == test_size def test_upload_data_if_table_exists_fail(self, project_id): @@ -725,7 +750,8 @@ def test_upload_data_if_table_exists_append(self, project_id): result = gbq.read_gbq("SELECT COUNT(*) AS num_rows FROM {0}" .format(self.destination_table + test_id), project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') assert result['num_rows'][0] == test_size * 2 # Try inserting with a different schema, confirm failure @@ -754,7 +780,8 @@ def test_upload_subset_columns_if_table_exists_append(self, project_id): result = gbq.read_gbq("SELECT COUNT(*) AS num_rows FROM {0}" .format(self.destination_table + test_id), project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') assert result['num_rows'][0] == test_size * 2 def test_upload_data_if_table_exists_replace(self, project_id): @@ -775,7 +802,8 @@ def test_upload_data_if_table_exists_replace(self, project_id): result = gbq.read_gbq("SELECT COUNT(*) AS num_rows FROM {0}" .format(self.destination_table + test_id), project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') assert result['num_rows'][0] == 5 def test_upload_data_if_table_exists_raises_value_error(self, project_id): @@ -818,7 +846,8 @@ def test_upload_chinese_unicode_data(self, project_id): result_df = gbq.read_gbq( "SELECT * FROM {0}".format(self.destination_table + test_id), project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') assert len(result_df) == test_size @@ -851,7 +880,8 @@ def test_upload_other_unicode_data(self, project_id): result_df = gbq.read_gbq("SELECT * FROM {0}".format( self.destination_table + test_id), project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') assert len(result_df) == test_size @@ -879,10 +909,11 @@ def test_upload_mixed_float_and_int(self, project_id): project_id=project_id, private_key=self.credentials) - result_df = gbq.read_gbq("SELECT * FROM {0}".format( - self.destination_table + test_id), + result_df = gbq.read_gbq( + 'SELECT * FROM {0}'.format(self.destination_table + test_id), project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') assert len(result_df) == test_size @@ -1173,10 +1204,11 @@ def test_upload_data_with_timestamp(self, project_id): project_id=project_id, private_key=self.credentials) - result_df = gbq.read_gbq("SELECT * FROM {0}".format( - self.destination_table + test_id), + result_df = gbq.read_gbq( + 'SELECT * FROM {0}'.format(self.destination_table + test_id), project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') assert len(result_df) == test_size diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index d3560450f054..f308d9f4aea9 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -190,12 +190,12 @@ def test_read_gbq_with_no_project_id_given_should_fail(monkeypatch): auth, 'get_application_default_credentials', mock_none_credentials) with pytest.raises(ValueError) as exception: - gbq.read_gbq('SELECT 1') + gbq.read_gbq('SELECT 1', dialect='standard') assert 'Could not determine project ID' in str(exception) def test_read_gbq_with_inferred_project_id(monkeypatch): - df = gbq.read_gbq('SELECT 1') + df = gbq.read_gbq('SELECT 1', dialect='standard') assert df is not None @@ -214,32 +214,35 @@ def test_that_parse_data_works_properly(): def test_read_gbq_with_invalid_private_key_json_should_fail(): with pytest.raises(pandas_gbq.exceptions.InvalidPrivateKeyFormat): - gbq.read_gbq('SELECT 1', project_id='x', private_key='y') + gbq.read_gbq( + 'SELECT 1', dialect='standard', project_id='x', private_key='y') def test_read_gbq_with_empty_private_key_json_should_fail(): with pytest.raises(pandas_gbq.exceptions.InvalidPrivateKeyFormat): - gbq.read_gbq('SELECT 1', project_id='x', private_key='{}') + gbq.read_gbq( + 'SELECT 1', dialect='standard', project_id='x', private_key='{}') def test_read_gbq_with_private_key_json_wrong_types_should_fail(): with pytest.raises(pandas_gbq.exceptions.InvalidPrivateKeyFormat): gbq.read_gbq( - 'SELECT 1', project_id='x', + 'SELECT 1', dialect='standard', project_id='x', private_key='{ "client_email" : 1, "private_key" : True }') def test_read_gbq_with_empty_private_key_file_should_fail(): with tm.ensure_clean() as empty_file_path: with pytest.raises(pandas_gbq.exceptions.InvalidPrivateKeyFormat): - gbq.read_gbq('SELECT 1', project_id='x', + gbq.read_gbq('SELECT 1', dialect='standard', project_id='x', private_key=empty_file_path) def test_read_gbq_with_corrupted_private_key_json_should_fail(): with pytest.raises(pandas_gbq.exceptions.InvalidPrivateKeyFormat): gbq.read_gbq( - 'SELECT 1', project_id='x', private_key='99999999999999999') + 'SELECT 1', dialect='standard', project_id='x', + private_key='99999999999999999') def test_read_gbq_with_verbose_new_pandas_warns_deprecation(min_bq_version): @@ -272,7 +275,7 @@ def test_read_gbq_wo_verbose_w_new_pandas_no_warnings(recwarn, min_bq_version): 'pkg_resources.Distribution.parsed_version', new_callable=mock.PropertyMock) as mock_version: mock_version.side_effect = [min_bq_version, pandas_version] - gbq.read_gbq('SELECT 1', project_id='my-project') + gbq.read_gbq('SELECT 1', project_id='my-project', dialect='standard') assert len(recwarn) == 0 @@ -283,10 +286,23 @@ def test_read_gbq_with_verbose_old_pandas_no_warnings(recwarn, min_bq_version): 'pkg_resources.Distribution.parsed_version', new_callable=mock.PropertyMock) as mock_version: mock_version.side_effect = [min_bq_version, pandas_version] - gbq.read_gbq('SELECT 1', project_id='my-project', verbose=True) + gbq.read_gbq( + 'SELECT 1', project_id='my-project', dialect='standard', + verbose=True) assert len(recwarn) == 0 +def test_read_gbq_with_invalid_dialect(): + with pytest.raises(ValueError) as excinfo: + gbq.read_gbq('SELECT 1', dialect='invalid') + assert 'is not valid for dialect' in str(excinfo.value) + + +def test_read_gbq_without_dialect_warns_future_change(): + with pytest.warns(FutureWarning): + gbq.read_gbq('SELECT 1') + + def test_generate_bq_schema_deprecated(): # 11121 Deprecation of generate_bq_schema with pytest.warns(FutureWarning): From 46139ab6322b5b7d9cf5ac038854b16eb5be3a62 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Wed, 22 Aug 2018 13:34:15 -0400 Subject: [PATCH 140/519] Faster dataframe construction (#128) * push df construction to pandas * lint * gitignore additions * `.astype` conversion * ignore warnings from Google Cloud SDK auth * handle repeated fields in schema * remove comment * whats new * remove duplicate test --- packages/pandas-gbq/.gitignore | 2 + packages/pandas-gbq/docs/source/changelog.rst | 10 +++++ packages/pandas-gbq/pandas_gbq/gbq.py | 38 +++++++++---------- packages/pandas-gbq/tests/system/test_gbq.py | 16 ++------ packages/pandas-gbq/tests/unit/test_gbq.py | 3 ++ 5 files changed, 38 insertions(+), 31 deletions(-) diff --git a/packages/pandas-gbq/.gitignore b/packages/pandas-gbq/.gitignore index 251cc50df0ba..f0dd6fbdeb5e 100644 --- a/packages/pandas-gbq/.gitignore +++ b/packages/pandas-gbq/.gitignore @@ -69,6 +69,8 @@ dist **/wheelhouse/* # coverage .coverage +.testmondata +.pytest_cache .nox # OS generated files # diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 47a43f9fba7f..f5895b708409 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,6 +1,16 @@ Changelog ========= +.. _changelog-0.6.1: + +0.6.1 / [unreleased] +-------------------- + +- Improved ``read_gbq`` performance and memory consumption by delegating + ``DataFrame`` construction to the Pandas library, radically reducing + the number of loops that execute in python + (:issue:`128`) + .. _changelog-0.6.0: 0.6.0 / 2018-08-15 diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index e7642ccd61a5..20e303bf571e 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -3,15 +3,14 @@ import os import time import warnings +from collections import OrderedDict from datetime import datetime import numpy as np from pandas import DataFrame -from pandas.compat import lzip from pandas_gbq.exceptions import AccessDenied - logger = logging.getLogger(__name__) @@ -444,29 +443,30 @@ def _get_credentials_file(): 'PANDAS_GBQ_CREDENTIALS_FILE') -def _parse_data(schema, rows): +def _parse_schema(schema_fields): # see: # http://pandas.pydata.org/pandas-docs/dev/missing_data.html # #missing-data-casting-rules-and-indexing dtype_map = {'FLOAT': np.dtype(float), 'TIMESTAMP': 'M8[ns]'} - fields = schema['fields'] - col_types = [field['type'] for field in fields] - col_names = [str(field['name']) for field in fields] - col_dtypes = [ - dtype_map.get(field['type'].upper(), object) - if field['mode'].lower() != 'repeated' - else object - for field in fields - ] - page_array = np.zeros((len(rows),), dtype=lzip(col_names, col_dtypes)) - for row_num, entries in enumerate(rows): - for col_num in range(len(col_types)): - field_value = entries[col_num] - page_array[row_num][col_num] = field_value - - return DataFrame(page_array, columns=col_names) + for field in schema_fields: + name = str(field['name']) + if field['mode'].upper() == 'REPEATED': + yield name, object + else: + dtype = dtype_map.get(field['type'].upper(), object) + yield name, dtype + + +def _parse_data(schema, rows): + + column_dtypes = OrderedDict(_parse_schema(schema['fields'])) + + df = DataFrame(data=(iter(r) for r in rows), columns=column_dtypes.keys()) + for column in df: + df[column] = df[column].astype(column_dtypes[column]) + return df def read_gbq(query, project_id=None, index_col=None, col_order=None, diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index bb91befb4c0e..7a00c745b4d7 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -13,7 +13,6 @@ from pandas_gbq import gbq - TABLE_ID = 'new_test' @@ -21,8 +20,7 @@ def _get_dataset_prefix_random(): return ''.join(['pandas_gbq_', str(randint(1, 100000))]) -@pytest.fixture(autouse=True, scope='module') -def _test_imports(): +def test_imports(): try: import pkg_resources # noqa except ImportError: @@ -143,13 +141,6 @@ def setup(self, project, credentials): project, private_key=credentials) self.credentials = credentials - def test_should_properly_handle_valid_strings(self, project_id): - query = 'SELECT "PI" AS valid_string' - df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials, - dialect='legacy') - tm.assert_frame_equal(df, DataFrame({'valid_string': ['PI']})) - def test_should_properly_handle_empty_strings(self, project_id): query = 'SELECT "" AS empty_string' df = gbq.read_gbq(query, project_id=project_id, @@ -392,7 +383,8 @@ def test_bad_project_id(self): with pytest.raises(gbq.GenericGBQException): gbq.read_gbq('SELCET * FROM [publicdata:samples.shakespeare]', project_id='not-my-project', - private_key=self.credentials) + private_key=self.credentials, + dialect='legacy') def test_bad_table_name(self, project_id): with pytest.raises(gbq.GenericGBQException): @@ -427,7 +419,7 @@ def test_zero_rows(self, project_id): ('is_bot', np.dtype(bool)), ('ts', 'M8[ns]')]) expected_result = DataFrame( page_array, columns=['title', 'id', 'is_bot', 'ts']) - tm.assert_frame_equal(df, expected_result) + tm.assert_frame_equal(df, expected_result, check_index_type=False) def test_legacy_sql(self, project_id): legacy_sql = "SELECT id FROM [publicdata.samples.wikipedia] LIMIT 10" diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index f308d9f4aea9..216d1451505a 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -13,6 +13,9 @@ except ImportError: # pragma: NO COVER from unittest import mock +pytestmark = pytest.mark.filter_warnings( + "ignore:credentials from Google Cloud SDK") + @pytest.fixture def min_bq_version(): From f3b823c15cfffeb013b55d6b0067fcddd54e9172 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Thu, 23 Aug 2018 18:16:26 -0400 Subject: [PATCH 141/519] Add test for single row & column (#200) * add test for single row & column * remove old comment --- packages/pandas-gbq/tests/system/test_gbq.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index 7a00c745b4d7..da3e6880b84c 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -421,6 +421,14 @@ def test_zero_rows(self, project_id): page_array, columns=['title', 'id', 'is_bot', 'ts']) tm.assert_frame_equal(df, expected_result, check_index_type=False) + def test_one_row_one_column(self, project_id): + df = gbq.read_gbq("SELECT 3 as v", + project_id=project_id, + private_key=self.credentials, + dialect='standard') + expected_result = DataFrame(dict(v=[3])) + tm.assert_frame_equal(df, expected_result) + def test_legacy_sql(self, project_id): legacy_sql = "SELECT id FROM [publicdata.samples.wikipedia] LIMIT 10" From 7f77357bb443a408d433744fbd7f85e2b11b1240 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Sat, 25 Aug 2018 13:37:49 -0400 Subject: [PATCH 142/519] Reduce verbosity of logging (#201) * reduce verbosity of logging * changelog --- packages/pandas-gbq/docs/source/changelog.rst | 2 ++ packages/pandas-gbq/pandas_gbq/gbq.py | 13 ++++--------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index f5895b708409..b4b534885ea7 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -10,6 +10,8 @@ Changelog ``DataFrame`` construction to the Pandas library, radically reducing the number of loops that execute in python (:issue:`128`) +- Reduced verbosity of logging from ``read_gbq``, particularly for short + queries. (:issue:`201`) .. _changelog-0.6.0: diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 20e303bf571e..99150dc843fd 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -240,12 +240,12 @@ def run_query(self, query, **kwargs): self._start_timer() try: - logger.info('Requesting query... ') + logger.debug('Requesting query... ') query_reply = self.client.query( query, job_config=bigquery.QueryJobConfig.from_api_repr(job_config), location=self.location) - logger.info('ok.\nQuery running...') + logger.info('Query running...') except (RefreshError, ValueError): if self.private_key: raise AccessDenied( @@ -258,7 +258,7 @@ def run_query(self, query, **kwargs): self.process_http_error(ex) job_id = query_reply.job_id - logger.info('Job ID: %s\nQuery running...' % job_id) + logger.debug('Job ID: %s' % job_id) while query_reply.state != 'DONE': self.log_elapsed_seconds(' Elapsed', 's. Waiting...') @@ -303,8 +303,7 @@ def run_query(self, query, **kwargs): for field in rows_iter.schema], } - # log basic query stats - logger.info('Got {} rows.\n'.format(total_rows)) + logger.debug('Got {} rows.\n'.format(total_rows)) return schema, result_rows @@ -314,7 +313,6 @@ def load_data( from pandas_gbq import load total_rows = len(dataframe) - logger.info("\n\n") try: chunks = load.load_chunks(self.client, dataframe, dataset_id, @@ -328,8 +326,6 @@ def load_data( except self.http_error as ex: self.process_http_error(ex) - logger.info("\n") - def schema(self, dataset_id, table_id): """Retrieve the schema of the table @@ -611,7 +607,6 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, connector.log_elapsed_seconds( 'Total time taken', datetime.now().strftime('s.\nFinished at %Y-%m-%d %H:%M:%S.'), - 0 ) return final_df From 57b2ed7f84c387677d6575e3afa084d8b4b88800 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Thu, 30 Aug 2018 15:47:14 -0400 Subject: [PATCH 143/519] Formatting with black (#204) * black config fixups and config * black reformatting * move stickler to black * add black logo * logo typo --- packages/pandas-gbq/.stickler.yml | 12 +- packages/pandas-gbq/README.rst | 4 +- packages/pandas-gbq/nox.py | 88 +- packages/pandas-gbq/pandas_gbq/__init__.py | 5 +- packages/pandas-gbq/pandas_gbq/auth.py | 121 +- packages/pandas-gbq/pandas_gbq/exceptions.py | 4 +- packages/pandas-gbq/pandas_gbq/gbq.py | 423 +++-- packages/pandas-gbq/pandas_gbq/load.py | 39 +- packages/pandas-gbq/pandas_gbq/schema.py | 26 +- packages/pandas-gbq/pyproject.toml | 7 + packages/pandas-gbq/setup.cfg | 6 +- packages/pandas-gbq/setup.py | 54 +- packages/pandas-gbq/tests/system/conftest.py | 37 +- packages/pandas-gbq/tests/system/test_auth.py | 41 +- packages/pandas-gbq/tests/system/test_gbq.py | 1528 ++++++++++------- packages/pandas-gbq/tests/unit/test_auth.py | 49 +- packages/pandas-gbq/tests/unit/test_gbq.py | 263 +-- packages/pandas-gbq/tests/unit/test_load.py | 18 +- packages/pandas-gbq/tests/unit/test_schema.py | 52 +- 19 files changed, 1670 insertions(+), 1107 deletions(-) create mode 100644 packages/pandas-gbq/pyproject.toml diff --git a/packages/pandas-gbq/.stickler.yml b/packages/pandas-gbq/.stickler.yml index 17cd4582f9e7..6ef8c8c06bba 100644 --- a/packages/pandas-gbq/.stickler.yml +++ b/packages/pandas-gbq/.stickler.yml @@ -1,10 +1,4 @@ linters: - flake8: - max-line-length: 79 - fixer: true - ignore: -files: - ignore: - - doc/**/*.py -fixers: - enable: true + black: + config: ./pyproject.toml + fixer: true \ No newline at end of file diff --git a/packages/pandas-gbq/README.rst b/packages/pandas-gbq/README.rst index d23565199aa1..87a1cbdce9c7 100644 --- a/packages/pandas-gbq/README.rst +++ b/packages/pandas-gbq/README.rst @@ -1,7 +1,7 @@ pandas-gbq ========== -|Build Status| |Version Status| |Coverage Status| +|Build Status| |Version Status| |Coverage Status| |Black Formatted| **pandas-gbq** is a package providing an interface to the Google BigQuery API from pandas @@ -43,3 +43,5 @@ See the `pandas-gbq documentation `_ for mor :target: https://pypi.python.org/pypi/pandas-gbq/ .. |Coverage Status| image:: https://img.shields.io/codecov/c/github/pydata/pandas-gbq.svg :target: https://codecov.io/gh/pydata/pandas-gbq/ +.. |Black Formatted| image:: https://img.shields.io/badge/code%20style-black-000000.svg + :target: https://github.com/ambv/black \ No newline at end of file diff --git a/packages/pandas-gbq/nox.py b/packages/pandas-gbq/nox.py index 53f3b8a6770a..9731e438d25b 100644 --- a/packages/pandas-gbq/nox.py +++ b/packages/pandas-gbq/nox.py @@ -8,101 +8,99 @@ import nox - PANDAS_PRE_WHEELS = ( - 'https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83' - '.ssl.cf2.rackcdn.com') + "https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83" + ".ssl.cf2.rackcdn.com" +) @nox.session def default(session): - session.install('mock', 'pytest', 'pytest-cov') - session.install('-e', '.') + session.install("mock", "pytest", "pytest-cov") + session.install("-e", ".") # Skip local auth tests on Travis. additional_args = list(session.posargs) - if 'TRAVIS_BUILD_DIR' in os.environ: - additional_args = additional_args + [ - '-m', - 'not local_auth', - ] + if "TRAVIS_BUILD_DIR" in os.environ: + additional_args = additional_args + ["-m", "not local_auth"] session.run( - 'pytest', - os.path.join('.', 'tests'), - '--quiet', - '--cov=pandas_gbq', - '--cov=tests.unit', - '--cov-report', - 'xml:/tmp/pytest-cov.xml', + "pytest", + os.path.join(".", "tests"), + "--quiet", + "--cov=pandas_gbq", + "--cov=tests.unit", + "--cov-report", + "xml:/tmp/pytest-cov.xml", *additional_args ) @nox.session def unit(session): - session.install('mock', 'pytest', 'pytest-cov') - session.install('-e', '.') + session.install("mock", "pytest", "pytest-cov") + session.install("-e", ".") session.run( - 'pytest', - os.path.join('.', 'tests', 'unit'), - '--quiet', - '--cov=pandas_gbq', - '--cov=tests.unit', - '--cov-report', - 'xml:/tmp/pytest-cov.xml', + "pytest", + os.path.join(".", "tests", "unit"), + "--quiet", + "--cov=pandas_gbq", + "--cov=tests.unit", + "--cov-report", + "xml:/tmp/pytest-cov.xml", *session.posargs ) @nox.session def test27(session): - session.interpreter = 'python2.7' + session.interpreter = "python2.7" session.install( - '-r', os.path.join('.', 'ci', 'requirements-2.7-0.19.2.pip')) + "-r", os.path.join(".", "ci", "requirements-2.7-0.19.2.pip") + ) default(session) @nox.session def test35(session): - session.interpreter = 'python3.5' + session.interpreter = "python3.5" session.install( - '-r', os.path.join('.', 'ci', 'requirements-3.5-0.18.1.pip')) + "-r", os.path.join(".", "ci", "requirements-3.5-0.18.1.pip") + ) default(session) @nox.session def test36(session): - session.interpreter = 'python3.6' + session.interpreter = "python3.6" session.install( - '-r', os.path.join('.', 'ci', 'requirements-3.6-0.20.1.conda')) + "-r", os.path.join(".", "ci", "requirements-3.6-0.20.1.conda") + ) default(session) @nox.session def test36master(session): - session.interpreter = 'python3.6' + session.interpreter = "python3.6" session.install( - '--pre', - '--upgrade', - '--timeout=60', - '-f', PANDAS_PRE_WHEELS, - 'pandas') + "--pre", "--upgrade", "--timeout=60", "-f", PANDAS_PRE_WHEELS, "pandas" + ) session.install( - '-r', os.path.join('.', 'ci', 'requirements-3.6-MASTER.pip')) + "-r", os.path.join(".", "ci", "requirements-3.6-MASTER.pip") + ) default(session) @nox.session def lint(session): - session.install('flake8') - session.run('flake8', 'pandas_gbq', 'tests', '-v') + session.install("flake8") + session.run("flake8", "pandas_gbq", "tests", "-v") @nox.session def cover(session): - session.interpreter = 'python3.5' + session.interpreter = "python3.5" - session.install('coverage', 'pytest-cov') - session.run('coverage', 'report', '--show-missing', '--fail-under=40') - session.run('coverage', 'erase') + session.install("coverage", "pytest-cov") + session.run("coverage", "report", "--show-missing", "--fail-under=40") + session.run("coverage", "erase") diff --git a/packages/pandas-gbq/pandas_gbq/__init__.py b/packages/pandas-gbq/pandas_gbq/__init__.py index c689b0d1d24e..febda7c65477 100644 --- a/packages/pandas-gbq/pandas_gbq/__init__.py +++ b/packages/pandas-gbq/pandas_gbq/__init__.py @@ -1,7 +1,8 @@ from .gbq import to_gbq, read_gbq # noqa from ._version import get_versions + versions = get_versions() -__version__ = versions.get('closest-tag', versions['version']) -__git_revision__ = versions['full-revisionid'] +__version__ = versions.get("closest-tag", versions["version"]) +__git_revision__ = versions["full-revisionid"] del get_versions, versions diff --git a/packages/pandas-gbq/pandas_gbq/auth.py b/packages/pandas-gbq/pandas_gbq/auth.py index fcf9bc8f3c14..c27d342e6aeb 100644 --- a/packages/pandas-gbq/pandas_gbq/auth.py +++ b/packages/pandas-gbq/pandas_gbq/auth.py @@ -9,29 +9,31 @@ import pandas_gbq.exceptions - logger = logging.getLogger(__name__) -SCOPES = ['https://www.googleapis.com/auth/bigquery'] +SCOPES = ["https://www.googleapis.com/auth/bigquery"] def get_credentials( - private_key=None, project_id=None, reauth=False, - auth_local_webserver=False): + private_key=None, project_id=None, reauth=False, auth_local_webserver=False +): if private_key: return get_service_account_credentials(private_key) # Try to retrieve Application Default Credentials credentials, default_project = get_application_default_credentials( - project_id=project_id) + project_id=project_id + ) if credentials: return credentials, default_project credentials = get_user_account_credentials( - project_id=project_id, reauth=reauth, - auth_local_webserver=auth_local_webserver) + project_id=project_id, + reauth=reauth, + auth_local_webserver=auth_local_webserver, + ) return credentials, project_id @@ -48,13 +50,13 @@ def get_service_account_credentials(private_key): else: # ugly hack: 'private_key' field has new lines inside, # they break json parser, but we need to preserve them - json_key = json.loads(private_key.replace('\n', ' ')) - json_key['private_key'] = json_key['private_key'].replace( - ' ', '\n') + json_key = json.loads(private_key.replace("\n", " ")) + json_key["private_key"] = json_key["private_key"].replace( + " ", "\n" + ) if pandas.compat.PY3: - json_key['private_key'] = bytes( - json_key['private_key'], 'UTF-8') + json_key["private_key"] = bytes(json_key["private_key"], "UTF-8") credentials = Credentials.from_service_account_info(json_key) credentials = credentials.with_scopes(SCOPES) @@ -63,16 +65,18 @@ def get_service_account_credentials(private_key): request = google.auth.transport.requests.Request() credentials.refresh(request) - return credentials, json_key.get('project_id') + return credentials, json_key.get("project_id") except (KeyError, ValueError, TypeError, AttributeError): raise pandas_gbq.exceptions.InvalidPrivateKeyFormat( - 'Detected private_key as {}. '.format( - 'path' if is_path else 'contents') + - 'Private key is missing or invalid. It should be service ' - 'account private key JSON (file path or string contents) ' + "Detected private_key as {}. ".format( + "path" if is_path else "contents" + ) + + "Private key is missing or invalid. It should be service " + "account private key JSON (file path or string contents) " 'with at least two keys: "client_email" and "private_key". ' - 'Can be obtained from: https://console.developers.google.' - 'com/permissions/serviceaccounts') + "Can be obtained from: https://console.developers.google." + "com/permissions/serviceaccounts" + ) def get_application_default_credentials(project_id=None): @@ -111,8 +115,11 @@ def get_application_default_credentials(project_id=None): def get_user_account_credentials( - project_id=None, reauth=False, auth_local_webserver=False, - credentials_path=None): + project_id=None, + reauth=False, + auth_local_webserver=False, + credentials_path=None, +): """Gets user account credentials. This method authenticates using user credentials, either loading saved @@ -140,26 +147,30 @@ def get_user_account_credentials( # current working directory. If the bigquery_credentials.dat file # exists in the current working directory, move the credentials to # the new default location. - if os.path.isfile('bigquery_credentials.dat'): - os.rename('bigquery_credentials.dat', credentials_path) + if os.path.isfile("bigquery_credentials.dat"): + os.rename("bigquery_credentials.dat", credentials_path) credentials = load_user_account_credentials( - project_id=project_id, credentials_path=credentials_path) + project_id=project_id, credentials_path=credentials_path + ) client_config = { - 'installed': { - 'client_id': ('495642085510-k0tmvj2m941jhre2nbqka17vqpjfddtd' - '.apps.googleusercontent.com'), - 'client_secret': 'kOc9wMptUtxkcIFbtZCcrEAc', - 'redirect_uris': ['urn:ietf:wg:oauth:2.0:oob'], - 'auth_uri': 'https://accounts.google.com/o/oauth2/auth', - 'token_uri': 'https://accounts.google.com/o/oauth2/token', + "installed": { + "client_id": ( + "495642085510-k0tmvj2m941jhre2nbqka17vqpjfddtd" + ".apps.googleusercontent.com" + ), + "client_secret": "kOc9wMptUtxkcIFbtZCcrEAc", + "redirect_uris": ["urn:ietf:wg:oauth:2.0:oob"], + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://accounts.google.com/o/oauth2/token", } } if credentials is None or reauth: app_flow = InstalledAppFlow.from_client_config( - client_config, scopes=SCOPES) + client_config, scopes=SCOPES + ) try: if auth_local_webserver: @@ -168,7 +179,8 @@ def get_user_account_credentials( credentials = app_flow.run_console() except OAuth2Error as ex: raise pandas_gbq.exceptions.AccessDenied( - "Unable to get valid credentials: {0}".format(ex)) + "Unable to get valid credentials: {0}".format(ex) + ) save_user_account_credentials(credentials, credentials_path) @@ -205,13 +217,14 @@ def load_user_account_credentials(project_id=None, credentials_path=None): return None credentials = Credentials( - token=credentials_json.get('access_token'), - refresh_token=credentials_json.get('refresh_token'), - id_token=credentials_json.get('id_token'), - token_uri=credentials_json.get('token_uri'), - client_id=credentials_json.get('client_id'), - client_secret=credentials_json.get('client_secret'), - scopes=credentials_json.get('scopes')) + token=credentials_json.get("access_token"), + refresh_token=credentials_json.get("refresh_token"), + id_token=credentials_json.get("id_token"), + token_uri=credentials_json.get("token_uri"), + client_id=credentials_json.get("client_id"), + client_secret=credentials_json.get("client_secret"), + scopes=credentials_json.get("scopes"), + ) # Refresh the token before trying to use it. request = google.auth.transport.requests.Request() @@ -230,19 +243,19 @@ def get_default_credentials_path(): ------- Path to the BigQuery credentials """ - if os.name == 'nt': - config_path = os.environ['APPDATA'] + if os.name == "nt": + config_path = os.environ["APPDATA"] else: - config_path = os.path.join(os.path.expanduser('~'), '.config') + config_path = os.path.join(os.path.expanduser("~"), ".config") - config_path = os.path.join(config_path, 'pandas_gbq') + config_path = os.path.join(config_path, "pandas_gbq") # Create a pandas_gbq directory in an application-specific hidden # user folder on the operating system. if not os.path.exists(config_path): os.makedirs(config_path) - return os.path.join(config_path, 'bigquery_credentials.dat') + return os.path.join(config_path, "bigquery_credentials.dat") def save_user_account_credentials(credentials, credentials_path): @@ -252,18 +265,18 @@ def save_user_account_credentials(credentials, credentials_path): .. versionadded 0.2.0 """ try: - with open(credentials_path, 'w') as credentials_file: + with open(credentials_path, "w") as credentials_file: credentials_json = { - 'refresh_token': credentials.refresh_token, - 'id_token': credentials.id_token, - 'token_uri': credentials.token_uri, - 'client_id': credentials.client_id, - 'client_secret': credentials.client_secret, - 'scopes': credentials.scopes, + "refresh_token": credentials.refresh_token, + "id_token": credentials.id_token, + "token_uri": credentials.token_uri, + "client_id": credentials.client_id, + "client_secret": credentials.client_secret, + "scopes": credentials.scopes, } json.dump(credentials_json, credentials_file) except IOError: - logger.warning('Unable to save credentials.') + logger.warning("Unable to save credentials.") def _try_credentials(project_id, credentials): @@ -278,7 +291,7 @@ def _try_credentials(project_id, credentials): try: client = bigquery.Client(project=project_id, credentials=credentials) # Check if the application has rights to the BigQuery project - client.query('SELECT 1').result() + client.query("SELECT 1").result() return credentials except google.api_core.exceptions.GoogleAPIError: return None diff --git a/packages/pandas-gbq/pandas_gbq/exceptions.py b/packages/pandas-gbq/pandas_gbq/exceptions.py index a8b6aca0526e..96711455fe46 100644 --- a/packages/pandas-gbq/pandas_gbq/exceptions.py +++ b/packages/pandas-gbq/pandas_gbq/exceptions.py @@ -1,9 +1,8 @@ - - class AccessDenied(ValueError): """ Raised when invalid credentials are provided, or tokens have expired. """ + pass @@ -11,4 +10,5 @@ class InvalidPrivateKeyFormat(ValueError): """ Raised when provided private key has invalid format. """ + pass diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 99150dc843fd..eba08009c800 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -30,26 +30,31 @@ def _check_google_client_version(): import pkg_resources except ImportError: - raise ImportError('Could not import pkg_resources (setuptools).') + raise ImportError("Could not import pkg_resources (setuptools).") # https://github.com/GoogleCloudPlatform/google-cloud-python/blob/master/bigquery/CHANGELOG.md - bigquery_minimum_version = pkg_resources.parse_version('0.32.0') + bigquery_minimum_version = pkg_resources.parse_version("0.32.0") BIGQUERY_INSTALLED_VERSION = pkg_resources.get_distribution( - 'google-cloud-bigquery').parsed_version + "google-cloud-bigquery" + ).parsed_version if BIGQUERY_INSTALLED_VERSION < bigquery_minimum_version: raise ImportError( - 'pandas-gbq requires google-cloud-bigquery >= {0}, ' - 'current version {1}'.format( - bigquery_minimum_version, BIGQUERY_INSTALLED_VERSION)) + "pandas-gbq requires google-cloud-bigquery >= {0}, " + "current version {1}".format( + bigquery_minimum_version, BIGQUERY_INSTALLED_VERSION + ) + ) # Add check for Pandas version before showing deprecation warning. # https://github.com/pydata/pandas-gbq/issues/157 pandas_installed_version = pkg_resources.get_distribution( - 'pandas').parsed_version - pandas_version_wo_verbosity = pkg_resources.parse_version('0.23.0') + "pandas" + ).parsed_version + pandas_version_wo_verbosity = pkg_resources.parse_version("0.23.0") SHOW_VERBOSE_DEPRECATION = ( - pandas_installed_version >= pandas_version_wo_verbosity) + pandas_installed_version >= pandas_version_wo_verbosity + ) def _test_google_api_imports(): @@ -58,19 +63,20 @@ def _test_google_api_imports(): from google_auth_oauthlib.flow import InstalledAppFlow # noqa except ImportError as ex: raise ImportError( - 'pandas-gbq requires google-auth-oauthlib: {0}'.format(ex)) + "pandas-gbq requires google-auth-oauthlib: {0}".format(ex) + ) try: import google.auth # noqa except ImportError as ex: - raise ImportError( - "pandas-gbq requires google-auth: {0}".format(ex)) + raise ImportError("pandas-gbq requires google-auth: {0}".format(ex)) try: from google.cloud import bigquery # noqa except ImportError as ex: raise ImportError( - "pandas-gbq requires google-cloud-bigquery: {0}".format(ex)) + "pandas-gbq requires google-cloud-bigquery: {0}".format(ex) + ) _check_google_client_version() @@ -79,6 +85,7 @@ class DatasetCreationError(ValueError): """ Raised when the create dataset method fails """ + pass @@ -86,6 +93,7 @@ class GenericGBQException(ValueError): """ Raised when an unrecognized Google API Error occurs. """ + pass @@ -95,6 +103,7 @@ class InvalidColumnOrder(ValueError): results DataFrame does not match the schema returned by BigQuery. """ + pass @@ -104,6 +113,7 @@ class InvalidIndexColumn(ValueError): results DataFrame does not match the schema returned by BigQuery. """ + pass @@ -112,6 +122,7 @@ class InvalidPageToken(ValueError): Raised when Google BigQuery fails to return, or returns a duplicate page token. """ + pass @@ -121,6 +132,7 @@ class InvalidSchema(ValueError): not match the schema of the destination table in BigQuery. """ + pass @@ -129,6 +141,7 @@ class NotFoundException(ValueError): Raised when the project_id, table or dataset provided in the query could not be found. """ + pass @@ -137,6 +150,7 @@ class QueryTimeout(ValueError): Raised when the query request exceeds the timeoutMs value specified in the BigQuery configuration. """ + pass @@ -144,17 +158,24 @@ class TableCreationError(ValueError): """ Raised when the create table method fails """ + pass class GbqConnector(object): - - def __init__(self, project_id, reauth=False, - private_key=None, auth_local_webserver=False, - dialect='legacy', location=None): + def __init__( + self, + project_id, + reauth=False, + private_key=None, + auth_local_webserver=False, + dialect="legacy", + location=None, + ): from google.api_core.exceptions import GoogleAPIError from google.api_core.exceptions import ClientError from pandas_gbq import auth + self.http_error = (ClientError, GoogleAPIError) self.project_id = project_id self.location = location @@ -164,21 +185,25 @@ def __init__(self, project_id, reauth=False, self.dialect = dialect self.credentials_path = _get_credentials_file() self.credentials, default_project = auth.get_credentials( - private_key=private_key, project_id=project_id, reauth=reauth, - auth_local_webserver=auth_local_webserver) + private_key=private_key, + project_id=project_id, + reauth=reauth, + auth_local_webserver=auth_local_webserver, + ) if self.project_id is None: self.project_id = default_project if self.project_id is None: raise ValueError( - 'Could not determine project ID and one was not supplied.') + "Could not determine project ID and one was not supplied." + ) self.client = self.get_client() # BQ Queries costs $5 per TB. First 1 TB per month is free # see here for more: https://cloud.google.com/bigquery/pricing - self.query_price_for_TB = 5. / 2**40 # USD/TB + self.query_price_for_TB = 5. / 2 ** 40 # USD/TB def _start_timer(self): self.start = time.time() @@ -186,26 +211,27 @@ def _start_timer(self): def get_elapsed_seconds(self): return round(time.time() - self.start, 2) - def log_elapsed_seconds(self, prefix='Elapsed', postfix='s.', - overlong=7): + def log_elapsed_seconds(self, prefix="Elapsed", postfix="s.", overlong=7): sec = self.get_elapsed_seconds() if sec > overlong: - logger.info('{} {} {}'.format(prefix, sec, postfix)) + logger.info("{} {} {}".format(prefix, sec, postfix)) # http://stackoverflow.com/questions/1094841/reusable-library-to-get-human-readable-version-of-file-size @staticmethod - def sizeof_fmt(num, suffix='B'): + def sizeof_fmt(num, suffix="B"): fmt = "%3.1f %s%s" - for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']: + for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]: if abs(num) < 1024.0: return fmt % (num, unit, suffix) num /= 1024.0 - return fmt % (num, 'Y', suffix) + return fmt % (num, "Y", suffix) def get_client(self): from google.cloud import bigquery + return bigquery.Client( - project=self.project_id, credentials=self.credentials) + project=self.project_id, credentials=self.credentials + ) @staticmethod def process_http_error(ex): @@ -220,52 +246,58 @@ def run_query(self, query, **kwargs): from google.cloud import bigquery job_config = { - 'query': { - 'useLegacySql': self.dialect == 'legacy' + "query": { + "useLegacySql": self.dialect + == "legacy" # 'allowLargeResults', 'createDisposition', # 'preserveNulls', destinationTable, useQueryCache } } - config = kwargs.get('configuration') + config = kwargs.get("configuration") if config is not None: job_config.update(config) - if 'query' in config and 'query' in config['query']: + if "query" in config and "query" in config["query"]: if query is not None: - raise ValueError("Query statement can't be specified " - "inside config while it is specified " - "as parameter") - query = config['query'].pop('query') + raise ValueError( + "Query statement can't be specified " + "inside config while it is specified " + "as parameter" + ) + query = config["query"].pop("query") self._start_timer() try: - logger.debug('Requesting query... ') + logger.debug("Requesting query... ") query_reply = self.client.query( query, job_config=bigquery.QueryJobConfig.from_api_repr(job_config), - location=self.location) - logger.info('Query running...') + location=self.location, + ) + logger.info("Query running...") except (RefreshError, ValueError): if self.private_key: raise AccessDenied( - "The service account credentials are not valid") + "The service account credentials are not valid" + ) else: raise AccessDenied( "The credentials have been revoked or expired, " - "please re-run the application to re-authorize") + "please re-run the application to re-authorize" + ) except self.http_error as ex: self.process_http_error(ex) job_id = query_reply.job_id - logger.debug('Job ID: %s' % job_id) + logger.debug("Job ID: %s" % job_id) - while query_reply.state != 'DONE': - self.log_elapsed_seconds(' Elapsed', 's. Waiting...') + while query_reply.state != "DONE": + self.log_elapsed_seconds(" Elapsed", "s. Waiting...") - timeout_ms = job_config['query'].get('timeoutMs') + timeout_ms = job_config["query"].get("timeoutMs") if timeout_ms and timeout_ms < self.get_elapsed_seconds() * 1000: - raise QueryTimeout('Query timeout: {} ms'.format(timeout_ms)) + raise QueryTimeout("Query timeout: {} ms".format(timeout_ms)) timeout_sec = 1.0 if timeout_ms: @@ -281,15 +313,21 @@ def run_query(self, query, **kwargs): self.process_http_error(ex) if query_reply.cache_hit: - logger.debug('Query done.\nCache hit.\n') + logger.debug("Query done.\nCache hit.\n") else: bytes_processed = query_reply.total_bytes_processed or 0 bytes_billed = query_reply.total_bytes_billed or 0 - logger.debug('Query done.\nProcessed: {} Billed: {}'.format( - self.sizeof_fmt(bytes_processed), - self.sizeof_fmt(bytes_billed))) - logger.debug('Standard price: ${:,.2f} USD\n'.format( - bytes_billed * self.query_price_for_TB)) + logger.debug( + "Query done.\nProcessed: {} Billed: {}".format( + self.sizeof_fmt(bytes_processed), + self.sizeof_fmt(bytes_billed), + ) + ) + logger.debug( + "Standard price: ${:,.2f} USD\n".format( + bytes_billed * self.query_price_for_TB + ) + ) try: rows_iter = query_reply.result() @@ -298,31 +336,44 @@ def run_query(self, query, **kwargs): result_rows = list(rows_iter) total_rows = rows_iter.total_rows schema = { - 'fields': [ - field.to_api_repr() - for field in rows_iter.schema], + "fields": [field.to_api_repr() for field in rows_iter.schema] } - logger.debug('Got {} rows.\n'.format(total_rows)) + logger.debug("Got {} rows.\n".format(total_rows)) return schema, result_rows def load_data( - self, dataframe, dataset_id, table_id, chunksize=None, - schema=None, progress_bar=True): + self, + dataframe, + dataset_id, + table_id, + chunksize=None, + schema=None, + progress_bar=True, + ): from pandas_gbq import load total_rows = len(dataframe) try: - chunks = load.load_chunks(self.client, dataframe, dataset_id, - table_id, chunksize=chunksize, - schema=schema, location=self.location) + chunks = load.load_chunks( + self.client, + dataframe, + dataset_id, + table_id, + chunksize=chunksize, + schema=schema, + location=self.location, + ) if progress_bar and tqdm: chunks = tqdm.tqdm(chunks) for remaining_rows in chunks: - logger.info("\rLoad is {0}% Complete".format( - ((total_rows - remaining_rows) * 100) / total_rows)) + logger.info( + "\rLoad is {0}% Complete".format( + ((total_rows - remaining_rows) * 100) / total_rows + ) + ) except self.http_error as ex: self.process_http_error(ex) @@ -351,10 +402,11 @@ def schema(self, dataset_id, table_id): remote_schema = table.schema remote_fields = [ - field_remote.to_api_repr() for field_remote in remote_schema] + field_remote.to_api_repr() for field_remote in remote_schema + ] for field in remote_fields: - field['type'] = field['type'].upper() - field['mode'] = field['mode'].upper() + field["type"] = field["type"].upper() + field["mode"] = field["mode"].upper() return remote_fields except self.http_error as ex: @@ -362,10 +414,10 @@ def schema(self, dataset_id, table_id): def _clean_schema_fields(self, fields): """Return a sanitized version of the schema for comparisons.""" - fields_sorted = sorted(fields, key=lambda field: field['name']) + fields_sorted = sorted(fields, key=lambda field: field["name"]) # Ignore mode and description when comparing schemas. return [ - {'name': field['name'], 'type': field['type']} + {"name": field["name"], "type": field["type"]} for field in fields_sorted ] @@ -393,8 +445,9 @@ def verify_schema(self, dataset_id, table_id, schema): """ fields_remote = self._clean_schema_fields( - self.schema(dataset_id, table_id)) - fields_local = self._clean_schema_fields(schema['fields']) + self.schema(dataset_id, table_id) + ) + fields_local = self._clean_schema_fields(schema["fields"]) return fields_remote == fields_local @@ -422,42 +475,42 @@ def schema_is_subset(self, dataset_id, table_id, schema): """ fields_remote = self._clean_schema_fields( - self.schema(dataset_id, table_id)) - fields_local = self._clean_schema_fields(schema['fields']) + self.schema(dataset_id, table_id) + ) + fields_local = self._clean_schema_fields(schema["fields"]) return all(field in fields_remote for field in fields_local) def delete_and_recreate_table(self, dataset_id, table_id, table_schema): - table = _Table(self.project_id, dataset_id, - private_key=self.private_key) + table = _Table( + self.project_id, dataset_id, private_key=self.private_key + ) table.delete(table_id) table.create(table_id, table_schema) def _get_credentials_file(): - return os.environ.get( - 'PANDAS_GBQ_CREDENTIALS_FILE') + return os.environ.get("PANDAS_GBQ_CREDENTIALS_FILE") def _parse_schema(schema_fields): # see: # http://pandas.pydata.org/pandas-docs/dev/missing_data.html # #missing-data-casting-rules-and-indexing - dtype_map = {'FLOAT': np.dtype(float), - 'TIMESTAMP': 'M8[ns]'} + dtype_map = {"FLOAT": np.dtype(float), "TIMESTAMP": "M8[ns]"} for field in schema_fields: - name = str(field['name']) - if field['mode'].upper() == 'REPEATED': + name = str(field["name"]) + if field["mode"].upper() == "REPEATED": yield name, object else: - dtype = dtype_map.get(field['type'].upper(), object) + dtype = dtype_map.get(field["type"].upper(), object) yield name, dtype def _parse_data(schema, rows): - column_dtypes = OrderedDict(_parse_schema(schema['fields'])) + column_dtypes = OrderedDict(_parse_schema(schema["fields"])) df = DataFrame(data=(iter(r) for r in rows), columns=column_dtypes.keys()) for column in df: @@ -465,10 +518,19 @@ def _parse_data(schema, rows): return df -def read_gbq(query, project_id=None, index_col=None, col_order=None, - reauth=False, private_key=None, auth_local_webserver=False, - dialect=None, location=None, configuration=None, - verbose=None): +def read_gbq( + query, + project_id=None, + index_col=None, + col_order=None, + reauth=False, + private_key=None, + auth_local_webserver=False, + dialect=None, + location=None, + configuration=None, + verbose=None, +): r"""Load data from Google BigQuery using google-cloud-python The main method a user calls to execute a Query in Google BigQuery @@ -551,12 +613,14 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, DataFrame representing results of query. """ if dialect is None: - dialect = 'legacy' + dialect = "legacy" warnings.warn( 'The default value for dialect is changing to "standard" in a ' 'future version. Pass in dialect="legacy" to disable this ' - 'warning.', - FutureWarning, stacklevel=2) + "warning.", + FutureWarning, + stacklevel=2, + ) _test_google_api_imports() @@ -564,14 +628,22 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, warnings.warn( "verbose is deprecated and will be removed in " "a future version. Set logging level in order to vary " - "verbosity", FutureWarning, stacklevel=2) + "verbosity", + FutureWarning, + stacklevel=2, + ) - if dialect not in ('legacy', 'standard'): + if dialect not in ("legacy", "standard"): raise ValueError("'{0}' is not valid for dialect".format(dialect)) connector = GbqConnector( - project_id, reauth=reauth, private_key=private_key, dialect=dialect, - auth_local_webserver=auth_local_webserver, location=location) + project_id, + reauth=reauth, + private_key=private_key, + dialect=dialect, + auth_local_webserver=auth_local_webserver, + location=location, + ) schema, rows = connector.run_query(query, configuration=configuration) final_df = _parse_data(schema, rows) @@ -581,8 +653,9 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, final_df.set_index(index_col, inplace=True) else: raise InvalidIndexColumn( - 'Index column "{0}" does not exist in DataFrame.' - .format(index_col) + 'Index column "{0}" does not exist in DataFrame.'.format( + index_col + ) ) # Change the order of columns in the DataFrame based on provided list @@ -591,31 +664,44 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None, final_df = final_df[col_order] else: raise InvalidColumnOrder( - 'Column order does not match this DataFrame.' + "Column order does not match this DataFrame." ) # cast BOOLEAN and INTEGER columns from object to bool/int # if they dont have any nulls AND field mode is not repeated (i.e., array) - type_map = {'BOOLEAN': bool, 'INTEGER': np.int64} - for field in schema['fields']: - if field['type'].upper() in type_map and \ - final_df[field['name']].notnull().all() and \ - field['mode'].lower() != 'repeated': - final_df[field['name']] = \ - final_df[field['name']].astype(type_map[field['type'].upper()]) + type_map = {"BOOLEAN": bool, "INTEGER": np.int64} + for field in schema["fields"]: + if ( + field["type"].upper() in type_map + and final_df[field["name"]].notnull().all() + and field["mode"].lower() != "repeated" + ): + final_df[field["name"]] = final_df[field["name"]].astype( + type_map[field["type"].upper()] + ) connector.log_elapsed_seconds( - 'Total time taken', - datetime.now().strftime('s.\nFinished at %Y-%m-%d %H:%M:%S.'), + "Total time taken", + datetime.now().strftime("s.\nFinished at %Y-%m-%d %H:%M:%S."), ) return final_df -def to_gbq(dataframe, destination_table, project_id=None, chunksize=None, - reauth=False, if_exists='fail', private_key=None, - auth_local_webserver=False, table_schema=None, location=None, - progress_bar=True, verbose=None): +def to_gbq( + dataframe, + destination_table, + project_id=None, + chunksize=None, + reauth=False, + if_exists="fail", + private_key=None, + auth_local_webserver=False, + table_schema=None, + location=None, + progress_bar=True, + verbose=None, +): """Write a DataFrame to a Google BigQuery table. The main method a user calls to export pandas DataFrame contents to @@ -699,22 +785,31 @@ def to_gbq(dataframe, destination_table, project_id=None, chunksize=None, warnings.warn( "verbose is deprecated and will be removed in " "a future version. Set logging level in order to vary " - "verbosity", FutureWarning, stacklevel=1) + "verbosity", + FutureWarning, + stacklevel=1, + ) - if if_exists not in ('fail', 'replace', 'append'): + if if_exists not in ("fail", "replace", "append"): raise ValueError("'{0}' is not valid for if_exists".format(if_exists)) - if '.' not in destination_table: + if "." not in destination_table: raise NotFoundException( - "Invalid Table Name. Should be of the form 'datasetId.tableId' ") + "Invalid Table Name. Should be of the form 'datasetId.tableId' " + ) connector = GbqConnector( - project_id, reauth=reauth, private_key=private_key, - auth_local_webserver=auth_local_webserver, location=location) - dataset_id, table_id = destination_table.rsplit('.', 1) + project_id, + reauth=reauth, + private_key=private_key, + auth_local_webserver=auth_local_webserver, + location=location, + ) + dataset_id, table_id = destination_table.rsplit(".", 1) - table = _Table(project_id, dataset_id, reauth=reauth, - private_key=private_key) + table = _Table( + project_id, dataset_id, reauth=reauth, private_key=private_key + ) if not table_schema: table_schema = _generate_bq_schema(dataframe) @@ -723,30 +818,40 @@ def to_gbq(dataframe, destination_table, project_id=None, chunksize=None, # If table exists, check if_exists parameter if table.exists(table_id): - if if_exists == 'fail': - raise TableCreationError("Could not create the table because it " - "already exists. " - "Change the if_exists parameter to " - "'append' or 'replace' data.") - elif if_exists == 'replace': + if if_exists == "fail": + raise TableCreationError( + "Could not create the table because it " + "already exists. " + "Change the if_exists parameter to " + "'append' or 'replace' data." + ) + elif if_exists == "replace": connector.delete_and_recreate_table( - dataset_id, table_id, table_schema) - elif if_exists == 'append': - if not connector.schema_is_subset(dataset_id, - table_id, - table_schema): - raise InvalidSchema("Please verify that the structure and " - "data types in the DataFrame match the " - "schema of the destination table.") + dataset_id, table_id, table_schema + ) + elif if_exists == "append": + if not connector.schema_is_subset( + dataset_id, table_id, table_schema + ): + raise InvalidSchema( + "Please verify that the structure and " + "data types in the DataFrame match the " + "schema of the destination table." + ) else: table.create(table_id, table_schema) connector.load_data( - dataframe, dataset_id, table_id, chunksize=chunksize, - schema=table_schema, progress_bar=progress_bar) + dataframe, + dataset_id, + table_id, + chunksize=chunksize, + schema=table_schema, + progress_bar=progress_bar, + ) -def generate_bq_schema(df, default_type='STRING'): +def generate_bq_schema(df, default_type="STRING"): """DEPRECATED: Given a passed df, generate the associated Google BigQuery schema. @@ -758,19 +863,23 @@ def generate_bq_schema(df, default_type='STRING'): does not exist in the schema. """ # deprecation TimeSeries, #11121 - warnings.warn("generate_bq_schema is deprecated and will be removed in " - "a future version", FutureWarning, stacklevel=2) + warnings.warn( + "generate_bq_schema is deprecated and will be removed in " + "a future version", + FutureWarning, + stacklevel=2, + ) return _generate_bq_schema(df, default_type=default_type) -def _generate_bq_schema(df, default_type='STRING'): +def _generate_bq_schema(df, default_type="STRING"): from pandas_gbq import schema + return schema.generate_bq_schema(df, default_type=default_type) class _Table(GbqConnector): - def __init__(self, project_id, dataset_id, reauth=False, private_key=None): self.dataset_id = dataset_id super(_Table, self).__init__(project_id, reauth, private_key) @@ -814,13 +923,16 @@ def create(self, table_id, schema): from google.cloud.bigquery import Table if self.exists(table_id): - raise TableCreationError("Table {0} already " - "exists".format(table_id)) + raise TableCreationError( + "Table {0} already " "exists".format(table_id) + ) - if not _Dataset(self.project_id, - private_key=self.private_key).exists(self.dataset_id): - _Dataset(self.project_id, - private_key=self.private_key).create(self.dataset_id) + if not _Dataset(self.project_id, private_key=self.private_key).exists( + self.dataset_id + ): + _Dataset(self.project_id, private_key=self.private_key).create( + self.dataset_id + ) table_ref = self.client.dataset(self.dataset_id).table(table_id) table = Table(table_ref) @@ -828,13 +940,12 @@ def create(self, table_id, schema): # Manually create the schema objects, adding NULLABLE mode # as a workaround for # https://github.com/GoogleCloudPlatform/google-cloud-python/issues/4456 - for field in schema['fields']: - if 'mode' not in field: - field['mode'] = 'NULLABLE' + for field in schema["fields"]: + if "mode" not in field: + field["mode"] = "NULLABLE" table.schema = [ - SchemaField.from_api_repr(field) - for field in schema['fields'] + SchemaField.from_api_repr(field) for field in schema["fields"] ] try: @@ -866,7 +977,6 @@ def delete(self, table_id): class _Dataset(GbqConnector): - def __init__(self, project_id, reauth=False, private_key=None): super(_Dataset, self).__init__(project_id, reauth, private_key) @@ -930,8 +1040,9 @@ def create(self, dataset_id): from google.cloud.bigquery import Dataset if self.exists(dataset_id): - raise DatasetCreationError("Dataset {0} already " - "exists".format(dataset_id)) + raise DatasetCreationError( + "Dataset {0} already " "exists".format(dataset_id) + ) dataset = Dataset(self.client.dataset(dataset_id)) @@ -952,7 +1063,8 @@ def delete(self, dataset_id): if not self.exists(dataset_id): raise NotFoundException( - "Dataset {0} does not exist".format(dataset_id)) + "Dataset {0} does not exist".format(dataset_id) + ) try: self.client.delete_dataset(self.client.dataset(dataset_id)) @@ -981,7 +1093,8 @@ def tables(self, dataset_id): try: table_response = self.client.list_tables( - self.client.dataset(dataset_id)) + self.client.dataset(dataset_id) + ) for row in table_response: table_list.append(row.table_id) diff --git a/packages/pandas-gbq/pandas_gbq/load.py b/packages/pandas-gbq/pandas_gbq/load.py index 436eb2d1706b..2cc0accbd817 100644 --- a/packages/pandas-gbq/pandas_gbq/load.py +++ b/packages/pandas-gbq/pandas_gbq/load.py @@ -14,15 +14,20 @@ def encode_chunk(dataframe): """ csv_buffer = six.StringIO() dataframe.to_csv( - csv_buffer, index=False, header=False, encoding='utf-8', - float_format='%.15g', date_format='%Y-%m-%d %H:%M:%S.%f') + csv_buffer, + index=False, + header=False, + encoding="utf-8", + float_format="%.15g", + date_format="%Y-%m-%d %H:%M:%S.%f", + ) # Convert to a BytesIO buffer so that unicode text is properly handled. # See: https://github.com/pydata/pandas-gbq/issues/106 body = csv_buffer.getvalue() if isinstance(body, bytes): - body = body.decode('utf-8') - body = body.encode('utf-8') + body = body.decode("utf-8") + body = body.encode("utf-8") return six.BytesIO(body) @@ -44,12 +49,18 @@ def encode_chunks(dataframe, chunksize=None): def load_chunks( - client, dataframe, dataset_id, table_id, chunksize=None, schema=None, - location=None): + client, + dataframe, + dataset_id, + table_id, + chunksize=None, + schema=None, + location=None, +): destination_table = client.dataset(dataset_id).table(table_id) job_config = bigquery.LoadJobConfig() - job_config.write_disposition = 'WRITE_APPEND' - job_config.source_format = 'CSV' + job_config.write_disposition = "WRITE_APPEND" + job_config.source_format = "CSV" if schema is None: schema = pandas_gbq.schema.generate_bq_schema(dataframe) @@ -57,13 +68,12 @@ def load_chunks( # Manually create the schema objects, adding NULLABLE mode # as a workaround for # https://github.com/GoogleCloudPlatform/google-cloud-python/issues/4456 - for field in schema['fields']: - if 'mode' not in field: - field['mode'] = 'NULLABLE' + for field in schema["fields"]: + if "mode" not in field: + field["mode"] = "NULLABLE" job_config.schema = [ - bigquery.SchemaField.from_api_repr(field) - for field in schema['fields'] + bigquery.SchemaField.from_api_repr(field) for field in schema["fields"] ] chunks = encode_chunks(dataframe, chunksize=chunksize) @@ -73,4 +83,5 @@ def load_chunks( chunk_buffer, destination_table, job_config=job_config, - location=location).result() + location=location, + ).result() diff --git a/packages/pandas-gbq/pandas_gbq/schema.py b/packages/pandas-gbq/pandas_gbq/schema.py index 25e3ca9ba358..3ca030252838 100644 --- a/packages/pandas-gbq/pandas_gbq/schema.py +++ b/packages/pandas-gbq/pandas_gbq/schema.py @@ -1,7 +1,7 @@ """Helper methods for BigQuery schemas""" -def generate_bq_schema(dataframe, default_type='STRING'): +def generate_bq_schema(dataframe, default_type="STRING"): """Given a passed dataframe, generate the associated Google BigQuery schema. Arguments: @@ -12,18 +12,22 @@ def generate_bq_schema(dataframe, default_type='STRING'): """ type_mapping = { - 'i': 'INTEGER', - 'b': 'BOOLEAN', - 'f': 'FLOAT', - 'O': 'STRING', - 'S': 'STRING', - 'U': 'STRING', - 'M': 'TIMESTAMP' + "i": "INTEGER", + "b": "BOOLEAN", + "f": "FLOAT", + "O": "STRING", + "S": "STRING", + "U": "STRING", + "M": "TIMESTAMP", } fields = [] for column_name, dtype in dataframe.dtypes.iteritems(): - fields.append({'name': column_name, - 'type': type_mapping.get(dtype.kind, default_type)}) + fields.append( + { + "name": column_name, + "type": type_mapping.get(dtype.kind, default_type), + } + ) - return {'fields': fields} + return {"fields": fields} diff --git a/packages/pandas-gbq/pyproject.toml b/packages/pandas-gbq/pyproject.toml new file mode 100644 index 000000000000..90440f599070 --- /dev/null +++ b/packages/pandas-gbq/pyproject.toml @@ -0,0 +1,7 @@ +[tool.black] +line-length = 79 +exclude = ''' +versioneer.py +| _version.py +| docs +''' \ No newline at end of file diff --git a/packages/pandas-gbq/setup.cfg b/packages/pandas-gbq/setup.cfg index 32e405568362..1f1185b3d6fb 100644 --- a/packages/pandas-gbq/setup.cfg +++ b/packages/pandas-gbq/setup.cfg @@ -12,9 +12,11 @@ tag_prefix = parentdir_prefix = pandas_gbq- [flake8] -ignore = E731 +ignore = E731, W503 +exclude = docs [isort] +multi_line_output=3 +line_length=79 default_section=THIRDPARTY known_first_party=pandas_gbq -multi_line_output=4 diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index bac3e5febfb6..61eb8691f4c9 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -4,7 +4,7 @@ import versioneer from setuptools import find_packages, setup -NAME = 'pandas-gbq' +NAME = "pandas-gbq" # versioning @@ -12,21 +12,19 @@ def readme(): - with open('README.rst') as f: + with open("README.rst") as f: return f.read() INSTALL_REQUIRES = [ - 'setuptools', - 'pandas', - 'google-auth', - 'google-auth-oauthlib', - 'google-cloud-bigquery>=0.32.0', + "setuptools", + "pandas", + "google-auth", + "google-auth-oauthlib", + "google-cloud-bigquery>=0.32.0", ] -extras = { - 'tqdm': 'tqdm>=4.23.0', -} +extras = {"tqdm": "tqdm>=4.23.0"} setup( name=NAME, @@ -34,26 +32,26 @@ def readme(): cmdclass=versioneer.get_cmdclass(), description="Pandas interface to Google BigQuery", long_description=readme(), - license='BSD License', - author='The PyData Development Team', - author_email='pydata@googlegroups.com', - url='https://github.com/pydata/pandas-gbq', + license="BSD License", + author="The PyData Development Team", + author_email="pydata@googlegroups.com", + url="https://github.com/pydata/pandas-gbq", classifiers=[ - 'Development Status :: 4 - Beta', - 'Environment :: Console', - 'Intended Audience :: Science/Research', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Topic :: Scientific/Engineering', + "Development Status :: 4 - Beta", + "Environment :: Console", + "Intended Audience :: Science/Research", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Topic :: Scientific/Engineering", ], - keywords='data', + keywords="data", install_requires=INSTALL_REQUIRES, extras_require=extras, - packages=find_packages(exclude=['contrib', 'docs', 'tests*']), - test_suite='tests', + packages=find_packages(exclude=["contrib", "docs", "tests*"]), + test_suite="tests", ) diff --git a/packages/pandas-gbq/tests/system/conftest.py b/packages/pandas-gbq/tests/system/conftest.py index fdd4d3f6b725..cd5a89be6b65 100644 --- a/packages/pandas-gbq/tests/system/conftest.py +++ b/packages/pandas-gbq/tests/system/conftest.py @@ -6,37 +6,42 @@ import pytest -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def project_id(): - return (os.environ.get('GBQ_PROJECT_ID') - or os.environ.get('GOOGLE_CLOUD_PROJECT')) # noqa + return os.environ.get("GBQ_PROJECT_ID") or os.environ.get( + "GOOGLE_CLOUD_PROJECT" + ) # noqa -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def private_key_path(): path = None - if 'TRAVIS_BUILD_DIR' in os.environ: + if "TRAVIS_BUILD_DIR" in os.environ: path = os.path.join( - os.environ['TRAVIS_BUILD_DIR'], 'ci', - 'travis_gbq.json') - elif 'GBQ_GOOGLE_APPLICATION_CREDENTIALS' in os.environ: - path = os.environ['GBQ_GOOGLE_APPLICATION_CREDENTIALS'] - elif 'GOOGLE_APPLICATION_CREDENTIALS' in os.environ: - path = os.environ['GOOGLE_APPLICATION_CREDENTIALS'] + os.environ["TRAVIS_BUILD_DIR"], "ci", "travis_gbq.json" + ) + elif "GBQ_GOOGLE_APPLICATION_CREDENTIALS" in os.environ: + path = os.environ["GBQ_GOOGLE_APPLICATION_CREDENTIALS"] + elif "GOOGLE_APPLICATION_CREDENTIALS" in os.environ: + path = os.environ["GOOGLE_APPLICATION_CREDENTIALS"] if path is None: - pytest.skip("Cannot run integration tests without a " - "private key json file path") + pytest.skip( + "Cannot run integration tests without a " + "private key json file path" + ) return None if not os.path.isfile(path): - pytest.skip("Cannot run integration tests when there is " - "no file at the private key json file path") + pytest.skip( + "Cannot run integration tests when there is " + "no file at the private key json file path" + ) return None return path -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def private_key_contents(private_key_path): if private_key_path is None: return None diff --git a/packages/pandas-gbq/tests/system/test_auth.py b/packages/pandas-gbq/tests/system/test_auth.py index c75a28d54e87..7ccc79c22fbc 100644 --- a/packages/pandas-gbq/tests/system/test_auth.py +++ b/packages/pandas-gbq/tests/system/test_auth.py @@ -21,7 +21,8 @@ def _check_if_can_get_correct_default_credentials(): try: credentials, project = google.auth.default( - scopes=pandas_gbq.auth.SCOPES) + scopes=pandas_gbq.auth.SCOPES + ) except (DefaultCredentialsError, IOError): return False @@ -30,23 +31,29 @@ def _check_if_can_get_correct_default_credentials(): def test_should_be_able_to_get_valid_credentials(project_id, private_key_path): credentials, _ = auth.get_credentials( - project_id=project_id, private_key=private_key_path) + project_id=project_id, private_key=private_key_path + ) assert credentials.valid def test_get_service_account_credentials_private_key_path(private_key_path): from google.auth.credentials import Credentials + credentials, project_id = auth.get_service_account_credentials( - private_key_path) + private_key_path + ) assert isinstance(credentials, Credentials) assert auth._try_credentials(project_id, credentials) is not None def test_get_service_account_credentials_private_key_contents( - private_key_contents): + private_key_contents +): from google.auth.credentials import Credentials + credentials, project_id = auth.get_service_account_credentials( - private_key_contents) + private_key_contents + ) assert isinstance(credentials, Credentials) assert auth._try_credentials(project_id, credentials) is not None @@ -55,8 +62,10 @@ def test_get_application_default_credentials_does_not_throw_error(): if _check_if_can_get_correct_default_credentials(): # Can get real credentials, so mock it out to fail. from google.auth.exceptions import DefaultCredentialsError - with mock.patch('google.auth.default', - side_effect=DefaultCredentialsError()): + + with mock.patch( + "google.auth.default", side_effect=DefaultCredentialsError() + ): credentials, _ = auth.get_application_default_credentials() else: credentials, _ = auth.get_application_default_credentials() @@ -65,9 +74,9 @@ def test_get_application_default_credentials_does_not_throw_error(): def test_get_application_default_credentials_returns_credentials(): if not _check_if_can_get_correct_default_credentials(): - pytest.skip("Cannot get default_credentials " - "from the environment!") + pytest.skip("Cannot get default_credentials " "from the environment!") from google.auth.credentials import Credentials + credentials, default_project = auth.get_application_default_credentials() assert isinstance(credentials, Credentials) @@ -77,7 +86,8 @@ def test_get_application_default_credentials_returns_credentials(): @pytest.mark.local_auth def test_get_user_account_credentials_bad_file_returns_credentials(): from google.auth.credentials import Credentials - with mock.patch('__main__.open', side_effect=IOError()): + + with mock.patch("__main__.open", side_effect=IOError()): credentials = auth.get_user_account_credentials() assert isinstance(credentials, Credentials) @@ -85,17 +95,18 @@ def test_get_user_account_credentials_bad_file_returns_credentials(): @pytest.mark.local_auth def test_get_user_account_credentials_returns_credentials(project_id): from google.auth.credentials import Credentials + credentials = auth.get_user_account_credentials( - project_id=project_id, - auth_local_webserver=True) + project_id=project_id, auth_local_webserver=True + ) assert isinstance(credentials, Credentials) @pytest.mark.local_auth def test_get_user_account_credentials_reauth_returns_credentials(project_id): from google.auth.credentials import Credentials + credentials = auth.get_user_account_credentials( - project_id=project_id, - auth_local_webserver=True, - reauth=True) + project_id=project_id, auth_local_webserver=True, reauth=True + ) assert isinstance(credentials, Credentials) diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index da3e6880b84c..261339f375ee 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -13,27 +13,27 @@ from pandas_gbq import gbq -TABLE_ID = 'new_test' +TABLE_ID = "new_test" def _get_dataset_prefix_random(): - return ''.join(['pandas_gbq_', str(randint(1, 100000))]) + return "".join(["pandas_gbq_", str(randint(1, 100000))]) def test_imports(): try: import pkg_resources # noqa except ImportError: - raise ImportError('Could not import pkg_resources (setuptools).') + raise ImportError("Could not import pkg_resources (setuptools).") gbq._test_google_api_imports() -@pytest.fixture(params=['env']) +@pytest.fixture(params=["env"]) def project(request, project_id): - if request.param == 'env': + if request.param == "env": return project_id - elif request.param == 'none': + elif request.param == "none": return None @@ -47,28 +47,31 @@ def gbq_connector(project, credentials): return gbq.GbqConnector(project, private_key=credentials) -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def bigquery_client(project_id, private_key_path): from google.cloud import bigquery + return bigquery.Client.from_service_account_json( - private_key_path, project=project_id) + private_key_path, project=project_id + ) -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def tokyo_dataset(bigquery_client): from google.cloud import bigquery - dataset_id = 'tokyo_{}'.format(_get_dataset_prefix_random()) + + dataset_id = "tokyo_{}".format(_get_dataset_prefix_random()) dataset_ref = bigquery_client.dataset(dataset_id) dataset = bigquery.Dataset(dataset_ref) - dataset.location = 'asia-northeast1' + dataset.location = "asia-northeast1" bigquery_client.create_dataset(dataset) yield dataset_id bigquery_client.delete_dataset(dataset_ref, delete_contents=True) -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def tokyo_table(bigquery_client, tokyo_dataset): - table_id = 'tokyo_table' + table_id = "tokyo_table" # Create a random table using DDL. # https://github.com/GoogleCloudPlatform/golang-samples/blob/2ab2c6b79a1ea3d71d8f91609b57a8fbde07ae5d/bigquery/snippets/snippet.go#L739 bigquery_client.query( @@ -77,8 +80,11 @@ def tokyo_table(bigquery_client, tokyo_dataset): 2000 + CAST(18 * RAND() as INT64) as year, IF(RAND() > 0.5,"foo","bar") as token FROM UNNEST(GENERATE_ARRAY(0,5,1)) as r - """.format(tokyo_dataset, table_id), - location='asia-northeast1').result() + """.format( + tokyo_dataset, table_id + ), + location="asia-northeast1", + ).result() return table_id @@ -95,337 +101,453 @@ def make_mixed_dataframe_v2(test_size): flts = np.random.randn(1, test_size) ints = np.random.randint(1, 10, size=(1, test_size)) strs = np.random.randint(1, 10, size=(1, test_size)).astype(str) - times = [datetime.now(pytz.timezone('US/Arizona')) - for t in range(test_size)] - return DataFrame({'bools': bools[0], - 'flts': flts[0], - 'ints': ints[0], - 'strs': strs[0], - 'times': times[0]}, - index=range(test_size)) + times = [ + datetime.now(pytz.timezone("US/Arizona")) for t in range(test_size) + ] + return DataFrame( + { + "bools": bools[0], + "flts": flts[0], + "ints": ints[0], + "strs": strs[0], + "times": times[0], + }, + index=range(test_size), + ) class TestGBQConnectorIntegration(object): - def test_should_be_able_to_make_a_connector(self, gbq_connector): - assert gbq_connector is not None, 'Could not create a GbqConnector' + assert gbq_connector is not None, "Could not create a GbqConnector" def test_should_be_able_to_get_a_bigquery_client(self, gbq_connector): bigquery_client = gbq_connector.get_client() assert bigquery_client is not None def test_should_be_able_to_get_schema_from_query(self, gbq_connector): - schema, pages = gbq_connector.run_query('SELECT 1') + schema, pages = gbq_connector.run_query("SELECT 1") assert schema is not None def test_should_be_able_to_get_results_from_query(self, gbq_connector): - schema, pages = gbq_connector.run_query('SELECT 1') + schema, pages = gbq_connector.run_query("SELECT 1") assert pages is not None def test_should_read(project, credentials): query = 'SELECT "PI" AS valid_string' df = gbq.read_gbq( - query, project_id=project, private_key=credentials, dialect='legacy') - tm.assert_frame_equal(df, DataFrame({'valid_string': ['PI']})) + query, project_id=project, private_key=credentials, dialect="legacy" + ) + tm.assert_frame_equal(df, DataFrame({"valid_string": ["PI"]})) class TestReadGBQIntegration(object): - @pytest.fixture(autouse=True) def setup(self, project, credentials): # - PER-TEST FIXTURES - # put here any instruction you want to be run *BEFORE* *EVERY* test is # executed. - self.gbq_connector = gbq.GbqConnector( - project, private_key=credentials) + self.gbq_connector = gbq.GbqConnector(project, private_key=credentials) self.credentials = credentials def test_should_properly_handle_empty_strings(self, project_id): query = 'SELECT "" AS empty_string' - df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials, - dialect='legacy') - tm.assert_frame_equal(df, DataFrame({'empty_string': [""]})) + df = gbq.read_gbq( + query, + project_id=project_id, + private_key=self.credentials, + dialect="legacy", + ) + tm.assert_frame_equal(df, DataFrame({"empty_string": [""]})) def test_should_properly_handle_null_strings(self, project_id): - query = 'SELECT STRING(NULL) AS null_string' - df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials, - dialect='legacy') - tm.assert_frame_equal(df, DataFrame({'null_string': [None]})) + query = "SELECT STRING(NULL) AS null_string" + df = gbq.read_gbq( + query, + project_id=project_id, + private_key=self.credentials, + dialect="legacy", + ) + tm.assert_frame_equal(df, DataFrame({"null_string": [None]})) def test_should_properly_handle_valid_integers(self, project_id): - query = 'SELECT INTEGER(3) AS valid_integer' - df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials, - dialect='legacy') - tm.assert_frame_equal(df, DataFrame({'valid_integer': [3]})) + query = "SELECT INTEGER(3) AS valid_integer" + df = gbq.read_gbq( + query, + project_id=project_id, + private_key=self.credentials, + dialect="legacy", + ) + tm.assert_frame_equal(df, DataFrame({"valid_integer": [3]})) def test_should_properly_handle_nullable_integers(self, project_id): - query = '''SELECT * FROM + query = """SELECT * FROM (SELECT 1 AS nullable_integer), - (SELECT NULL AS nullable_integer)''' - df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials, - dialect='legacy') + (SELECT NULL AS nullable_integer)""" + df = gbq.read_gbq( + query, + project_id=project_id, + private_key=self.credentials, + dialect="legacy", + ) tm.assert_frame_equal( - df, DataFrame({'nullable_integer': [1, None]}).astype(object)) + df, DataFrame({"nullable_integer": [1, None]}).astype(object) + ) def test_should_properly_handle_valid_longs(self, project_id): - query = 'SELECT 1 << 62 AS valid_long' - df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials, - dialect='legacy') - tm.assert_frame_equal( - df, DataFrame({'valid_long': [1 << 62]})) + query = "SELECT 1 << 62 AS valid_long" + df = gbq.read_gbq( + query, + project_id=project_id, + private_key=self.credentials, + dialect="legacy", + ) + tm.assert_frame_equal(df, DataFrame({"valid_long": [1 << 62]})) def test_should_properly_handle_nullable_longs(self, project_id): - query = '''SELECT * FROM + query = """SELECT * FROM (SELECT 1 << 62 AS nullable_long), - (SELECT NULL AS nullable_long)''' - df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials, - dialect='legacy') + (SELECT NULL AS nullable_long)""" + df = gbq.read_gbq( + query, + project_id=project_id, + private_key=self.credentials, + dialect="legacy", + ) tm.assert_frame_equal( - df, DataFrame({'nullable_long': [1 << 62, None]}).astype(object)) + df, DataFrame({"nullable_long": [1 << 62, None]}).astype(object) + ) def test_should_properly_handle_null_integers(self, project_id): - query = 'SELECT INTEGER(NULL) AS null_integer' - df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials, - dialect='legacy') - tm.assert_frame_equal(df, DataFrame({'null_integer': [None]})) + query = "SELECT INTEGER(NULL) AS null_integer" + df = gbq.read_gbq( + query, + project_id=project_id, + private_key=self.credentials, + dialect="legacy", + ) + tm.assert_frame_equal(df, DataFrame({"null_integer": [None]})) def test_should_properly_handle_valid_floats(self, project_id): from math import pi - query = 'SELECT PI() AS valid_float' - df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials, - dialect='legacy') - tm.assert_frame_equal(df, DataFrame( - {'valid_float': [pi]})) + + query = "SELECT PI() AS valid_float" + df = gbq.read_gbq( + query, + project_id=project_id, + private_key=self.credentials, + dialect="legacy", + ) + tm.assert_frame_equal(df, DataFrame({"valid_float": [pi]})) def test_should_properly_handle_nullable_floats(self, project_id): from math import pi - query = '''SELECT * FROM + + query = """SELECT * FROM (SELECT PI() AS nullable_float), - (SELECT NULL AS nullable_float)''' - df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials, - dialect='legacy') - tm.assert_frame_equal( - df, DataFrame({'nullable_float': [pi, None]})) + (SELECT NULL AS nullable_float)""" + df = gbq.read_gbq( + query, + project_id=project_id, + private_key=self.credentials, + dialect="legacy", + ) + tm.assert_frame_equal(df, DataFrame({"nullable_float": [pi, None]})) def test_should_properly_handle_valid_doubles(self, project_id): from math import pi - query = 'SELECT PI() * POW(10, 307) AS valid_double' - df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials, - dialect='legacy') - tm.assert_frame_equal(df, DataFrame( - {'valid_double': [pi * 10 ** 307]})) + + query = "SELECT PI() * POW(10, 307) AS valid_double" + df = gbq.read_gbq( + query, + project_id=project_id, + private_key=self.credentials, + dialect="legacy", + ) + tm.assert_frame_equal( + df, DataFrame({"valid_double": [pi * 10 ** 307]}) + ) def test_should_properly_handle_nullable_doubles(self, project_id): from math import pi - query = '''SELECT * FROM + + query = """SELECT * FROM (SELECT PI() * POW(10, 307) AS nullable_double), - (SELECT NULL AS nullable_double)''' - df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials, - dialect='legacy') + (SELECT NULL AS nullable_double)""" + df = gbq.read_gbq( + query, + project_id=project_id, + private_key=self.credentials, + dialect="legacy", + ) tm.assert_frame_equal( - df, DataFrame({'nullable_double': [pi * 10 ** 307, None]})) + df, DataFrame({"nullable_double": [pi * 10 ** 307, None]}) + ) def test_should_properly_handle_null_floats(self, project_id): - query = 'SELECT FLOAT(NULL) AS null_float' - df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials, - dialect='legacy') - tm.assert_frame_equal(df, DataFrame({'null_float': [np.nan]})) + query = "SELECT FLOAT(NULL) AS null_float" + df = gbq.read_gbq( + query, + project_id=project_id, + private_key=self.credentials, + dialect="legacy", + ) + tm.assert_frame_equal(df, DataFrame({"null_float": [np.nan]})) def test_should_properly_handle_timestamp_unix_epoch(self, project_id): query = 'SELECT TIMESTAMP("1970-01-01 00:00:00") AS unix_epoch' - df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials, - dialect='legacy') - tm.assert_frame_equal(df, DataFrame( - {'unix_epoch': [np.datetime64('1970-01-01T00:00:00.000000Z')]})) + df = gbq.read_gbq( + query, + project_id=project_id, + private_key=self.credentials, + dialect="legacy", + ) + tm.assert_frame_equal( + df, + DataFrame( + {"unix_epoch": [np.datetime64("1970-01-01T00:00:00.000000Z")]} + ), + ) def test_should_properly_handle_arbitrary_timestamp(self, project_id): query = 'SELECT TIMESTAMP("2004-09-15 05:00:00") AS valid_timestamp' - df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials, - dialect='legacy') - tm.assert_frame_equal(df, DataFrame({ - 'valid_timestamp': [np.datetime64('2004-09-15T05:00:00.000000Z')] - })) + df = gbq.read_gbq( + query, + project_id=project_id, + private_key=self.credentials, + dialect="legacy", + ) + tm.assert_frame_equal( + df, + DataFrame( + { + "valid_timestamp": [ + np.datetime64("2004-09-15T05:00:00.000000Z") + ] + } + ), + ) def test_should_properly_handle_null_timestamp(self, project_id): - query = 'SELECT TIMESTAMP(NULL) AS null_timestamp' - df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials, - dialect='legacy') - tm.assert_frame_equal(df, DataFrame({'null_timestamp': [NaT]})) + query = "SELECT TIMESTAMP(NULL) AS null_timestamp" + df = gbq.read_gbq( + query, + project_id=project_id, + private_key=self.credentials, + dialect="legacy", + ) + tm.assert_frame_equal(df, DataFrame({"null_timestamp": [NaT]})) def test_should_properly_handle_true_boolean(self, project_id): - query = 'SELECT BOOLEAN(TRUE) AS true_boolean' - df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials, - dialect='legacy') - tm.assert_frame_equal(df, DataFrame({'true_boolean': [True]})) + query = "SELECT BOOLEAN(TRUE) AS true_boolean" + df = gbq.read_gbq( + query, + project_id=project_id, + private_key=self.credentials, + dialect="legacy", + ) + tm.assert_frame_equal(df, DataFrame({"true_boolean": [True]})) def test_should_properly_handle_false_boolean(self, project_id): - query = 'SELECT BOOLEAN(FALSE) AS false_boolean' - df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials, - dialect='legacy') - tm.assert_frame_equal(df, DataFrame({'false_boolean': [False]})) + query = "SELECT BOOLEAN(FALSE) AS false_boolean" + df = gbq.read_gbq( + query, + project_id=project_id, + private_key=self.credentials, + dialect="legacy", + ) + tm.assert_frame_equal(df, DataFrame({"false_boolean": [False]})) def test_should_properly_handle_null_boolean(self, project_id): - query = 'SELECT BOOLEAN(NULL) AS null_boolean' - df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials, - dialect='legacy') - tm.assert_frame_equal(df, DataFrame({'null_boolean': [None]})) + query = "SELECT BOOLEAN(NULL) AS null_boolean" + df = gbq.read_gbq( + query, + project_id=project_id, + private_key=self.credentials, + dialect="legacy", + ) + tm.assert_frame_equal(df, DataFrame({"null_boolean": [None]})) def test_should_properly_handle_nullable_booleans(self, project_id): - query = '''SELECT * FROM + query = """SELECT * FROM (SELECT BOOLEAN(TRUE) AS nullable_boolean), - (SELECT NULL AS nullable_boolean)''' - df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials, - dialect='legacy') + (SELECT NULL AS nullable_boolean)""" + df = gbq.read_gbq( + query, + project_id=project_id, + private_key=self.credentials, + dialect="legacy", + ) tm.assert_frame_equal( - df, DataFrame({'nullable_boolean': [True, None]}).astype(object)) + df, DataFrame({"nullable_boolean": [True, None]}).astype(object) + ) def test_unicode_string_conversion_and_normalization(self, project_id): - correct_test_datatype = DataFrame( - {'unicode_string': [u("\xe9\xfc")]} - ) + correct_test_datatype = DataFrame({"unicode_string": [u("\xe9\xfc")]}) unicode_string = "\xc3\xa9\xc3\xbc" if compat.PY3: - unicode_string = unicode_string.encode('latin-1').decode('utf8') + unicode_string = unicode_string.encode("latin-1").decode("utf8") query = 'SELECT "{0}" AS unicode_string'.format(unicode_string) - df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials, - dialect='legacy') + df = gbq.read_gbq( + query, + project_id=project_id, + private_key=self.credentials, + dialect="legacy", + ) tm.assert_frame_equal(df, correct_test_datatype) def test_index_column(self, project_id): query = "SELECT 'a' AS string_1, 'b' AS string_2" - result_frame = gbq.read_gbq(query, project_id=project_id, - index_col="string_1", - private_key=self.credentials, - dialect='legacy') + result_frame = gbq.read_gbq( + query, + project_id=project_id, + index_col="string_1", + private_key=self.credentials, + dialect="legacy", + ) correct_frame = DataFrame( - {'string_1': ['a'], 'string_2': ['b']}).set_index("string_1") + {"string_1": ["a"], "string_2": ["b"]} + ).set_index("string_1") assert result_frame.index.name == correct_frame.index.name def test_column_order(self, project_id): query = "SELECT 'a' AS string_1, 'b' AS string_2, 'c' AS string_3" - col_order = ['string_3', 'string_1', 'string_2'] - result_frame = gbq.read_gbq(query, project_id=project_id, - col_order=col_order, - private_key=self.credentials, - dialect='legacy') - correct_frame = DataFrame({'string_1': ['a'], 'string_2': [ - 'b'], 'string_3': ['c']})[col_order] + col_order = ["string_3", "string_1", "string_2"] + result_frame = gbq.read_gbq( + query, + project_id=project_id, + col_order=col_order, + private_key=self.credentials, + dialect="legacy", + ) + correct_frame = DataFrame( + {"string_1": ["a"], "string_2": ["b"], "string_3": ["c"]} + )[col_order] tm.assert_frame_equal(result_frame, correct_frame) def test_read_gbq_raises_invalid_column_order(self, project_id): query = "SELECT 'a' AS string_1, 'b' AS string_2, 'c' AS string_3" - col_order = ['string_aaa', 'string_1', 'string_2'] + col_order = ["string_aaa", "string_1", "string_2"] # Column string_aaa does not exist. Should raise InvalidColumnOrder with pytest.raises(gbq.InvalidColumnOrder): - gbq.read_gbq(query, project_id=project_id, - col_order=col_order, - private_key=self.credentials, - dialect='legacy') + gbq.read_gbq( + query, + project_id=project_id, + col_order=col_order, + private_key=self.credentials, + dialect="legacy", + ) def test_column_order_plus_index(self, project_id): query = "SELECT 'a' AS string_1, 'b' AS string_2, 'c' AS string_3" - col_order = ['string_3', 'string_2'] - result_frame = gbq.read_gbq(query, project_id=project_id, - index_col='string_1', col_order=col_order, - private_key=self.credentials, - dialect='legacy') + col_order = ["string_3", "string_2"] + result_frame = gbq.read_gbq( + query, + project_id=project_id, + index_col="string_1", + col_order=col_order, + private_key=self.credentials, + dialect="legacy", + ) correct_frame = DataFrame( - {'string_1': ['a'], 'string_2': ['b'], 'string_3': ['c']}) - correct_frame.set_index('string_1', inplace=True) + {"string_1": ["a"], "string_2": ["b"], "string_3": ["c"]} + ) + correct_frame.set_index("string_1", inplace=True) correct_frame = correct_frame[col_order] tm.assert_frame_equal(result_frame, correct_frame) def test_read_gbq_raises_invalid_index_column(self, project_id): query = "SELECT 'a' AS string_1, 'b' AS string_2, 'c' AS string_3" - col_order = ['string_3', 'string_2'] + col_order = ["string_3", "string_2"] # Column string_bbb does not exist. Should raise InvalidIndexColumn with pytest.raises(gbq.InvalidIndexColumn): - gbq.read_gbq(query, project_id=project_id, - index_col='string_bbb', col_order=col_order, - private_key=self.credentials, - dialect='legacy') + gbq.read_gbq( + query, + project_id=project_id, + index_col="string_bbb", + col_order=col_order, + private_key=self.credentials, + dialect="legacy", + ) def test_malformed_query(self, project_id): with pytest.raises(gbq.GenericGBQException): - gbq.read_gbq("SELCET * FORM [publicdata:samples.shakespeare]", - project_id=project_id, - private_key=self.credentials, - dialect='legacy') + gbq.read_gbq( + "SELCET * FORM [publicdata:samples.shakespeare]", + project_id=project_id, + private_key=self.credentials, + dialect="legacy", + ) def test_bad_project_id(self): with pytest.raises(gbq.GenericGBQException): - gbq.read_gbq('SELCET * FROM [publicdata:samples.shakespeare]', - project_id='not-my-project', - private_key=self.credentials, - dialect='legacy') + gbq.read_gbq( + "SELCET * FROM [publicdata:samples.shakespeare]", + project_id="not-my-project", + private_key=self.credentials, + dialect="legacy", + ) def test_bad_table_name(self, project_id): with pytest.raises(gbq.GenericGBQException): - gbq.read_gbq("SELECT * FROM [publicdata:samples.nope]", - project_id=project_id, - private_key=self.credentials, - dialect='legacy') + gbq.read_gbq( + "SELECT * FROM [publicdata:samples.nope]", + project_id=project_id, + private_key=self.credentials, + dialect="legacy", + ) def test_download_dataset_larger_than_200k_rows(self, project_id): test_size = 200005 # Test for known BigQuery bug in datasets larger than 100k rows # http://stackoverflow.com/questions/19145587/bq-py-not-paging-results - df = gbq.read_gbq("SELECT id FROM [publicdata:samples.wikipedia] " - "GROUP EACH BY id ORDER BY id ASC LIMIT {0}" - .format(test_size), - project_id=project_id, - private_key=self.credentials, - dialect='legacy') + df = gbq.read_gbq( + "SELECT id FROM [publicdata:samples.wikipedia] " + "GROUP EACH BY id ORDER BY id ASC LIMIT {0}".format(test_size), + project_id=project_id, + private_key=self.credentials, + dialect="legacy", + ) assert len(df.drop_duplicates()) == test_size def test_zero_rows(self, project_id): # Bug fix for https://github.com/pandas-dev/pandas/issues/10273 - df = gbq.read_gbq("SELECT title, id, is_bot, " - "SEC_TO_TIMESTAMP(timestamp) ts " - "FROM [publicdata:samples.wikipedia] " - "WHERE timestamp=-9999999", - project_id=project_id, - private_key=self.credentials, - dialect='legacy') + df = gbq.read_gbq( + "SELECT title, id, is_bot, " + "SEC_TO_TIMESTAMP(timestamp) ts " + "FROM [publicdata:samples.wikipedia] " + "WHERE timestamp=-9999999", + project_id=project_id, + private_key=self.credentials, + dialect="legacy", + ) page_array = np.zeros( - (0,), dtype=[('title', object), ('id', np.dtype(int)), - ('is_bot', np.dtype(bool)), ('ts', 'M8[ns]')]) + (0,), + dtype=[ + ("title", object), + ("id", np.dtype(int)), + ("is_bot", np.dtype(bool)), + ("ts", "M8[ns]"), + ], + ) expected_result = DataFrame( - page_array, columns=['title', 'id', 'is_bot', 'ts']) + page_array, columns=["title", "id", "is_bot", "ts"] + ) tm.assert_frame_equal(df, expected_result, check_index_type=False) def test_one_row_one_column(self, project_id): - df = gbq.read_gbq("SELECT 3 as v", - project_id=project_id, - private_key=self.credentials, - dialect='standard') + df = gbq.read_gbq( + "SELECT 3 as v", + project_id=project_id, + private_key=self.credentials, + dialect="standard", + ) expected_result = DataFrame(dict(v=[3])) tm.assert_frame_equal(df, expected_result) @@ -435,158 +557,171 @@ def test_legacy_sql(self, project_id): # Test that a legacy sql statement fails when # setting dialect='standard' with pytest.raises(gbq.GenericGBQException): - gbq.read_gbq(legacy_sql, project_id=project_id, - dialect='standard', - private_key=self.credentials) + gbq.read_gbq( + legacy_sql, + project_id=project_id, + dialect="standard", + private_key=self.credentials, + ) # Test that a legacy sql statement succeeds when # setting dialect='legacy' - df = gbq.read_gbq(legacy_sql, project_id=project_id, - dialect='legacy', - private_key=self.credentials) + df = gbq.read_gbq( + legacy_sql, + project_id=project_id, + dialect="legacy", + private_key=self.credentials, + ) assert len(df.drop_duplicates()) == 10 def test_standard_sql(self, project_id): - standard_sql = "SELECT DISTINCT id FROM " \ - "`publicdata.samples.wikipedia` LIMIT 10" + standard_sql = ( + "SELECT DISTINCT id FROM " + "`publicdata.samples.wikipedia` LIMIT 10" + ) # Test that a standard sql statement fails when using # the legacy SQL dialect. with pytest.raises(gbq.GenericGBQException): - gbq.read_gbq(standard_sql, project_id=project_id, - private_key=self.credentials, - dialect='legacy') + gbq.read_gbq( + standard_sql, + project_id=project_id, + private_key=self.credentials, + dialect="legacy", + ) # Test that a standard sql statement succeeds when # setting dialect='standard' - df = gbq.read_gbq(standard_sql, project_id=project_id, - dialect='standard', - private_key=self.credentials) + df = gbq.read_gbq( + standard_sql, + project_id=project_id, + dialect="standard", + private_key=self.credentials, + ) assert len(df.drop_duplicates()) == 10 def test_query_with_parameters(self, project_id): sql_statement = "SELECT @param1 + @param2 AS valid_result" config = { - 'query': { + "query": { "useLegacySql": False, "parameterMode": "named", "queryParameters": [ { "name": "param1", - "parameterType": { - "type": "INTEGER" - }, - "parameterValue": { - "value": 1 - } + "parameterType": {"type": "INTEGER"}, + "parameterValue": {"value": 1}, }, { "name": "param2", - "parameterType": { - "type": "INTEGER" - }, - "parameterValue": { - "value": 2 - } - } - ] + "parameterType": {"type": "INTEGER"}, + "parameterValue": {"value": 2}, + }, + ], } } # Test that a query that relies on parameters fails # when parameters are not supplied via configuration with pytest.raises(ValueError): - gbq.read_gbq(sql_statement, project_id=project_id, - private_key=self.credentials, - dialect='legacy') + gbq.read_gbq( + sql_statement, + project_id=project_id, + private_key=self.credentials, + dialect="legacy", + ) # Test that the query is successful because we have supplied # the correct query parameters via the 'config' option - df = gbq.read_gbq(sql_statement, project_id=project_id, - private_key=self.credentials, - configuration=config, - dialect='legacy') - tm.assert_frame_equal(df, DataFrame({'valid_result': [3]})) + df = gbq.read_gbq( + sql_statement, + project_id=project_id, + private_key=self.credentials, + configuration=config, + dialect="legacy", + ) + tm.assert_frame_equal(df, DataFrame({"valid_result": [3]})) def test_query_inside_configuration(self, project_id): query_no_use = 'SELECT "PI_WRONG" AS valid_string' query = 'SELECT "PI" AS valid_string' - config = { - 'query': { - "query": query, - "useQueryCache": False, - } - } + config = {"query": {"query": query, "useQueryCache": False}} # Test that it can't pass query both # inside config and as parameter with pytest.raises(ValueError): - gbq.read_gbq(query_no_use, project_id=project_id, - private_key=self.credentials, - configuration=config, - dialect='legacy') + gbq.read_gbq( + query_no_use, + project_id=project_id, + private_key=self.credentials, + configuration=config, + dialect="legacy", + ) - df = gbq.read_gbq(None, project_id=project_id, - private_key=self.credentials, - configuration=config, - dialect='legacy') - tm.assert_frame_equal(df, DataFrame({'valid_string': ['PI']})) + df = gbq.read_gbq( + None, + project_id=project_id, + private_key=self.credentials, + configuration=config, + dialect="legacy", + ) + tm.assert_frame_equal(df, DataFrame({"valid_string": ["PI"]})) def test_configuration_without_query(self, project_id): - sql_statement = 'SELECT 1' + sql_statement = "SELECT 1" config = { - 'copy': { + "copy": { "sourceTable": { "projectId": project_id, "datasetId": "publicdata:samples", - "tableId": "wikipedia" + "tableId": "wikipedia", }, "destinationTable": { "projectId": project_id, "datasetId": "publicdata:samples", - "tableId": "wikipedia_copied" + "tableId": "wikipedia_copied", }, } } # Test that only 'query' configurations are supported # nor 'copy','load','extract' with pytest.raises(ValueError): - gbq.read_gbq(sql_statement, project_id=project_id, - private_key=self.credentials, - configuration=config, - dialect='legacy') + gbq.read_gbq( + sql_statement, + project_id=project_id, + private_key=self.credentials, + configuration=config, + dialect="legacy", + ) def test_configuration_raises_value_error_with_multiple_config( - self, project_id): - sql_statement = 'SELECT 1' + self, project_id + ): + sql_statement = "SELECT 1" config = { - 'query': { - "query": sql_statement, - "useQueryCache": False, - }, - 'load': { - "query": sql_statement, - "useQueryCache": False, - } + "query": {"query": sql_statement, "useQueryCache": False}, + "load": {"query": sql_statement, "useQueryCache": False}, } # Test that only ValueError is raised with multiple configurations with pytest.raises(ValueError): - gbq.read_gbq(sql_statement, project_id=project_id, - private_key=self.credentials, - configuration=config, - dialect='legacy') + gbq.read_gbq( + sql_statement, + project_id=project_id, + private_key=self.credentials, + configuration=config, + dialect="legacy", + ) def test_timeout_configuration(self, project_id): - sql_statement = 'SELECT 1' - config = { - 'query': { - "timeoutMs": 1 - } - } + sql_statement = "SELECT 1" + config = {"query": {"timeoutMs": 1}} # Test that QueryTimeout error raises with pytest.raises(gbq.QueryTimeout): - gbq.read_gbq(sql_statement, project_id=project_id, - private_key=self.credentials, - configuration=config, - dialect='legacy') + gbq.read_gbq( + sql_statement, + project_id=project_id, + private_key=self.credentials, + configuration=config, + dialect="legacy", + ) def test_query_response_bytes(self): assert self.gbq_connector.sizeof_fmt(999) == "999.0 B" @@ -606,20 +741,30 @@ def test_query_response_bytes(self): def test_struct(self, project_id): query = """SELECT 1 int_field, STRUCT("a" as letter, 1 as num) struct_field""" - df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials, - dialect='standard') - expected = DataFrame([[1, {"letter": "a", "num": 1}]], - columns=["int_field", "struct_field"]) + df = gbq.read_gbq( + query, + project_id=project_id, + private_key=self.credentials, + dialect="standard", + ) + expected = DataFrame( + [[1, {"letter": "a", "num": 1}]], + columns=["int_field", "struct_field"], + ) tm.assert_frame_equal(df, expected) def test_array(self, project_id): query = """select ["a","x","b","y","c","z"] as letters""" - df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials, - dialect='standard') - tm.assert_frame_equal(df, DataFrame([[["a", "x", "b", "y", "c", "z"]]], - columns=["letters"])) + df = gbq.read_gbq( + query, + project_id=project_id, + private_key=self.credentials, + dialect="standard", + ) + tm.assert_frame_equal( + df, + DataFrame([[["a", "x", "b", "y", "c", "z"]]], columns=["letters"]), + ) def test_array_length_zero(self, project_id): query = """WITH t as ( @@ -630,11 +775,16 @@ def test_array_length_zero(self, project_id): select letter, array_field, array_length(array_field) len from t order by letter ASC""" - df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials, - dialect='standard') - expected = DataFrame([["a", [""], 1], ["b", [], 0]], - columns=["letter", "array_field", "len"]) + df = gbq.read_gbq( + query, + project_id=project_id, + private_key=self.credentials, + dialect="standard", + ) + expected = DataFrame( + [["a", [""], 1], ["b", [], 0]], + columns=["letter", "array_field", "len"], + ) tm.assert_frame_equal(df, expected) def test_array_agg(self, project_id): @@ -649,29 +799,42 @@ def test_array_agg(self, project_id): from t group by letter order by letter ASC""" - df = gbq.read_gbq(query, project_id=project_id, - private_key=self.credentials, - dialect='standard') - tm.assert_frame_equal(df, DataFrame([["a", [1, 3]], ["b", [2]]], - columns=["letter", "numbers"])) + df = gbq.read_gbq( + query, + project_id=project_id, + private_key=self.credentials, + dialect="standard", + ) + tm.assert_frame_equal( + df, + DataFrame( + [["a", [1, 3]], ["b", [2]]], columns=["letter", "numbers"] + ), + ) def test_array_of_floats(self, private_key_path, project_id): query = """select [1.1, 2.2, 3.3] as a, 4 as b""" - df = gbq.read_gbq(query, project_id=project_id, - private_key=private_key_path, - dialect='standard') - tm.assert_frame_equal(df, DataFrame([[[1.1, 2.2, 3.3], 4]], - columns=["a", "b"])) + df = gbq.read_gbq( + query, + project_id=project_id, + private_key=private_key_path, + dialect="standard", + ) + tm.assert_frame_equal( + df, DataFrame([[[1.1, 2.2, 3.3], 4]], columns=["a", "b"]) + ) def test_tokyo(self, tokyo_dataset, tokyo_table, private_key_path): df = gbq.read_gbq( - 'SELECT MAX(year) AS max_year FROM {}.{}'.format( - tokyo_dataset, tokyo_table), - dialect='standard', - location='asia-northeast1', - private_key=private_key_path) + "SELECT MAX(year) AS max_year FROM {}.{}".format( + tokyo_dataset, tokyo_table + ), + dialect="standard", + location="asia-northeast1", + private_key=private_key_path, + ) print(df) - assert df['max_year'][0] >= 2000 + assert df["max_year"][0] >= 2000 class TestToGBQIntegration(object): @@ -681,7 +844,7 @@ class TestToGBQIntegration(object): # test is added See `Issue 191 # `__ - @pytest.fixture(autouse=True, scope='function') + @pytest.fixture(autouse=True, scope="function") def setup(self, project, credentials, bigquery_client): # - PER-TEST FIXTURES - # put here any instruction you want to be run *BEFORE* *EVERY* test is @@ -689,14 +852,14 @@ def setup(self, project, credentials, bigquery_client): self.dataset_prefix = _get_dataset_prefix_random() clean_gbq_environment(self.dataset_prefix, bigquery_client) - self.dataset = gbq._Dataset(project, - private_key=credentials) - self.table = gbq._Table(project, self.dataset_prefix + "1", - private_key=credentials) - self.sut = gbq.GbqConnector(project, - private_key=credentials) - self.destination_table = "{0}{1}.{2}".format(self.dataset_prefix, "1", - TABLE_ID) + self.dataset = gbq._Dataset(project, private_key=credentials) + self.table = gbq._Table( + project, self.dataset_prefix + "1", private_key=credentials + ) + self.sut = gbq.GbqConnector(project, private_key=credentials) + self.destination_table = "{0}{1}.{2}".format( + self.dataset_prefix, "1", TABLE_ID + ) self.dataset.create(self.dataset_prefix + "1") self.credentials = credentials yield @@ -707,15 +870,23 @@ def test_upload_data(self, project_id): test_size = 20001 df = make_mixed_dataframe_v2(test_size) - gbq.to_gbq(df, self.destination_table + test_id, project_id, - chunksize=10000, private_key=self.credentials) + gbq.to_gbq( + df, + self.destination_table + test_id, + project_id, + chunksize=10000, + private_key=self.credentials, + ) - result = gbq.read_gbq("SELECT COUNT(*) AS num_rows FROM {0}" - .format(self.destination_table + test_id), - project_id=project_id, - private_key=self.credentials, - dialect='legacy') - assert result['num_rows'][0] == test_size + result = gbq.read_gbq( + "SELECT COUNT(*) AS num_rows FROM {0}".format( + self.destination_table + test_id + ), + project_id=project_id, + private_key=self.credentials, + dialect="legacy", + ) + assert result["num_rows"][0] == test_size def test_upload_data_if_table_exists_fail(self, project_id): test_id = "2" @@ -725,13 +896,22 @@ def test_upload_data_if_table_exists_fail(self, project_id): # Test the default value of if_exists is 'fail' with pytest.raises(gbq.TableCreationError): - gbq.to_gbq(df, self.destination_table + test_id, project_id, - private_key=self.credentials) + gbq.to_gbq( + df, + self.destination_table + test_id, + project_id, + private_key=self.credentials, + ) # Test the if_exists parameter with value 'fail' with pytest.raises(gbq.TableCreationError): - gbq.to_gbq(df, self.destination_table + test_id, project_id, - if_exists='fail', private_key=self.credentials) + gbq.to_gbq( + df, + self.destination_table + test_id, + project_id, + if_exists="fail", + private_key=self.credentials, + ) def test_upload_data_if_table_exists_append(self, project_id): test_id = "3" @@ -740,25 +920,42 @@ def test_upload_data_if_table_exists_append(self, project_id): df_different_schema = tm.makeMixedDataFrame() # Initialize table with sample data - gbq.to_gbq(df, self.destination_table + test_id, project_id, - chunksize=10000, private_key=self.credentials) + gbq.to_gbq( + df, + self.destination_table + test_id, + project_id, + chunksize=10000, + private_key=self.credentials, + ) # Test the if_exists parameter with value 'append' - gbq.to_gbq(df, self.destination_table + test_id, project_id, - if_exists='append', private_key=self.credentials) + gbq.to_gbq( + df, + self.destination_table + test_id, + project_id, + if_exists="append", + private_key=self.credentials, + ) - result = gbq.read_gbq("SELECT COUNT(*) AS num_rows FROM {0}" - .format(self.destination_table + test_id), - project_id=project_id, - private_key=self.credentials, - dialect='legacy') - assert result['num_rows'][0] == test_size * 2 + result = gbq.read_gbq( + "SELECT COUNT(*) AS num_rows FROM {0}".format( + self.destination_table + test_id + ), + project_id=project_id, + private_key=self.credentials, + dialect="legacy", + ) + assert result["num_rows"][0] == test_size * 2 # Try inserting with a different schema, confirm failure with pytest.raises(gbq.InvalidSchema): - gbq.to_gbq(df_different_schema, self.destination_table + test_id, - project_id, if_exists='append', - private_key=self.credentials) + gbq.to_gbq( + df_different_schema, + self.destination_table + test_id, + project_id, + if_exists="append", + private_key=self.credentials, + ) def test_upload_subset_columns_if_table_exists_append(self, project_id): # Issue 24: Upload is succesful if dataframe has columns @@ -769,20 +966,32 @@ def test_upload_subset_columns_if_table_exists_append(self, project_id): df_subset_cols = df.iloc[:, :2] # Initialize table with sample data - gbq.to_gbq(df, self.destination_table + test_id, project_id, - chunksize=10000, private_key=self.credentials) + gbq.to_gbq( + df, + self.destination_table + test_id, + project_id, + chunksize=10000, + private_key=self.credentials, + ) # Test the if_exists parameter with value 'append' - gbq.to_gbq(df_subset_cols, - self.destination_table + test_id, project_id, - if_exists='append', private_key=self.credentials) + gbq.to_gbq( + df_subset_cols, + self.destination_table + test_id, + project_id, + if_exists="append", + private_key=self.credentials, + ) - result = gbq.read_gbq("SELECT COUNT(*) AS num_rows FROM {0}" - .format(self.destination_table + test_id), - project_id=project_id, - private_key=self.credentials, - dialect='legacy') - assert result['num_rows'][0] == test_size * 2 + result = gbq.read_gbq( + "SELECT COUNT(*) AS num_rows FROM {0}".format( + self.destination_table + test_id + ), + project_id=project_id, + private_key=self.credentials, + dialect="legacy", + ) + assert result["num_rows"][0] == test_size * 2 def test_upload_data_if_table_exists_replace(self, project_id): test_id = "4" @@ -791,20 +1000,32 @@ def test_upload_data_if_table_exists_replace(self, project_id): df_different_schema = tm.makeMixedDataFrame() # Initialize table with sample data - gbq.to_gbq(df, self.destination_table + test_id, project_id, - chunksize=10000, private_key=self.credentials) + gbq.to_gbq( + df, + self.destination_table + test_id, + project_id, + chunksize=10000, + private_key=self.credentials, + ) # Test the if_exists parameter with the value 'replace'. - gbq.to_gbq(df_different_schema, self.destination_table + test_id, - project_id, if_exists='replace', - private_key=self.credentials) + gbq.to_gbq( + df_different_schema, + self.destination_table + test_id, + project_id, + if_exists="replace", + private_key=self.credentials, + ) - result = gbq.read_gbq("SELECT COUNT(*) AS num_rows FROM {0}" - .format(self.destination_table + test_id), - project_id=project_id, - private_key=self.credentials, - dialect='legacy') - assert result['num_rows'][0] == 5 + result = gbq.read_gbq( + "SELECT COUNT(*) AS num_rows FROM {0}".format( + self.destination_table + test_id + ), + project_id=project_id, + private_key=self.credentials, + dialect="legacy", + ) + assert result["num_rows"][0] == 5 def test_upload_data_if_table_exists_raises_value_error(self, project_id): test_id = "4" @@ -813,83 +1034,108 @@ def test_upload_data_if_table_exists_raises_value_error(self, project_id): # Test invalid value for if_exists parameter raises value error with pytest.raises(ValueError): - gbq.to_gbq(df, self.destination_table + test_id, project_id, - if_exists='xxxxx', private_key=self.credentials) + gbq.to_gbq( + df, + self.destination_table + test_id, + project_id, + if_exists="xxxxx", + private_key=self.credentials, + ) def test_google_upload_errors_should_raise_exception(self, project_id): raise pytest.skip("buggy test") test_id = "5" - test_timestamp = datetime.now(pytz.timezone('US/Arizona')) - bad_df = DataFrame({'bools': [False, False], 'flts': [0.0, 1.0], - 'ints': [0, '1'], 'strs': ['a', 1], - 'times': [test_timestamp, test_timestamp]}, - index=range(2)) + test_timestamp = datetime.now(pytz.timezone("US/Arizona")) + bad_df = DataFrame( + { + "bools": [False, False], + "flts": [0.0, 1.0], + "ints": [0, "1"], + "strs": ["a", 1], + "times": [test_timestamp, test_timestamp], + }, + index=range(2), + ) with pytest.raises(gbq.StreamingInsertError): - gbq.to_gbq(bad_df, self.destination_table + test_id, - project_id, private_key=self.credentials) + gbq.to_gbq( + bad_df, + self.destination_table + test_id, + project_id, + private_key=self.credentials, + ) def test_upload_chinese_unicode_data(self, project_id): test_id = "2" test_size = 6 - df = DataFrame(np.random.randn(6, 4), index=range(6), - columns=list('ABCD')) - df['s'] = u'信用卡' + df = DataFrame( + np.random.randn(6, 4), index=range(6), columns=list("ABCD") + ) + df["s"] = u"信用卡" gbq.to_gbq( - df, self.destination_table + test_id, + df, + self.destination_table + test_id, project_id, private_key=self.credentials, - chunksize=10000) + chunksize=10000, + ) result_df = gbq.read_gbq( "SELECT * FROM {0}".format(self.destination_table + test_id), project_id=project_id, private_key=self.credentials, - dialect='legacy') + dialect="legacy", + ) assert len(result_df) == test_size if sys.version_info.major < 3: - pytest.skip(msg='Unicode comparison in Py2 not working') + pytest.skip(msg="Unicode comparison in Py2 not working") - result = result_df['s'].sort_values() - expected = df['s'].sort_values() + result = result_df["s"].sort_values() + expected = df["s"].sort_values() tm.assert_numpy_array_equal(expected.values, result.values) def test_upload_other_unicode_data(self, project_id): test_id = "3" test_size = 3 - df = DataFrame({ - 's': ['Skywalker™', 'lego', 'hülle'], - 'i': [200, 300, 400], - 'd': [ - '2017-12-13 17:40:39', '2017-12-13 17:40:39', - '2017-12-13 17:40:39' - ] - }) + df = DataFrame( + { + "s": ["Skywalker™", "lego", "hülle"], + "i": [200, 300, 400], + "d": [ + "2017-12-13 17:40:39", + "2017-12-13 17:40:39", + "2017-12-13 17:40:39", + ], + } + ) gbq.to_gbq( - df, self.destination_table + test_id, + df, + self.destination_table + test_id, project_id=project_id, private_key=self.credentials, - chunksize=10000) + chunksize=10000, + ) - result_df = gbq.read_gbq("SELECT * FROM {0}".format( - self.destination_table + test_id), + result_df = gbq.read_gbq( + "SELECT * FROM {0}".format(self.destination_table + test_id), project_id=project_id, private_key=self.credentials, - dialect='legacy') + dialect="legacy", + ) assert len(result_df) == test_size if sys.version_info.major < 3: - pytest.skip(msg='Unicode comparison in Py2 not working') + pytest.skip(msg="Unicode comparison in Py2 not working") - result = result_df['s'].sort_values() - expected = df['s'].sort_values() + result = result_df["s"].sort_values() + expected = df["s"].sort_values() tm.assert_numpy_array_equal(expected.values, result.values) @@ -901,19 +1147,23 @@ def test_upload_mixed_float_and_int(self, project_id): test_size = 2 df = DataFrame( [[1, 1.1], [2, 2.2]], - index=['row 1', 'row 2'], - columns=['intColumn', 'floatColumn']) + index=["row 1", "row 2"], + columns=["intColumn", "floatColumn"], + ) gbq.to_gbq( - df, self.destination_table + test_id, + df, + self.destination_table + test_id, project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + ) result_df = gbq.read_gbq( - 'SELECT * FROM {0}'.format(self.destination_table + test_id), + "SELECT * FROM {0}".format(self.destination_table + test_id), project_id=project_id, private_key=self.credentials, - dialect='legacy') + dialect="legacy", + ) assert len(result_df) == test_size @@ -922,10 +1172,14 @@ def test_generate_schema(self): df = tm.makeMixedDataFrame() schema = gbq._generate_bq_schema(df) - test_schema = {'fields': [{'name': 'A', 'type': 'FLOAT'}, - {'name': 'B', 'type': 'FLOAT'}, - {'name': 'C', 'type': 'STRING'}, - {'name': 'D', 'type': 'TIMESTAMP'}]} + test_schema = { + "fields": [ + {"name": "A", "type": "FLOAT"}, + {"name": "B", "type": "FLOAT"}, + {"name": "C", "type": "STRING"}, + {"name": "D", "type": "TIMESTAMP"}, + ] + } assert schema == test_schema @@ -948,14 +1202,17 @@ def test_table_does_not_exist(self): def test_delete_table(self): test_id = "8" - test_schema = {'fields': [{'name': 'A', 'type': 'FLOAT'}, - {'name': 'B', 'type': 'FLOAT'}, - {'name': 'C', 'type': 'STRING'}, - {'name': 'D', 'type': 'TIMESTAMP'}]} + test_schema = { + "fields": [ + {"name": "A", "type": "FLOAT"}, + {"name": "B", "type": "FLOAT"}, + {"name": "C", "type": "STRING"}, + {"name": "D", "type": "TIMESTAMP"}, + ] + } self.table.create(TABLE_ID + test_id, test_schema) self.table.delete(TABLE_ID + test_id) - assert not self.table.exists( - TABLE_ID + test_id) + assert not self.table.exists(TABLE_ID + test_id) def test_delete_table_not_found(self): with pytest.raises(gbq.NotFoundException): @@ -963,58 +1220,90 @@ def test_delete_table_not_found(self): def test_list_table(self): test_id = "9" - test_schema = {'fields': [{'name': 'A', 'type': 'FLOAT'}, - {'name': 'B', 'type': 'FLOAT'}, - {'name': 'C', 'type': 'STRING'}, - {'name': 'D', 'type': 'TIMESTAMP'}]} + test_schema = { + "fields": [ + {"name": "A", "type": "FLOAT"}, + {"name": "B", "type": "FLOAT"}, + {"name": "C", "type": "STRING"}, + {"name": "D", "type": "TIMESTAMP"}, + ] + } self.table.create(TABLE_ID + test_id, test_schema) assert TABLE_ID + test_id in self.dataset.tables( - self.dataset_prefix + "1") + self.dataset_prefix + "1" + ) def test_verify_schema_allows_flexible_column_order(self): test_id = "10" - test_schema_1 = {'fields': [{'name': 'A', 'type': 'FLOAT'}, - {'name': 'B', 'type': 'FLOAT'}, - {'name': 'C', 'type': 'STRING'}, - {'name': 'D', 'type': 'TIMESTAMP'}]} - test_schema_2 = {'fields': [{'name': 'A', 'type': 'FLOAT'}, - {'name': 'C', 'type': 'STRING'}, - {'name': 'B', 'type': 'FLOAT'}, - {'name': 'D', 'type': 'TIMESTAMP'}]} + test_schema_1 = { + "fields": [ + {"name": "A", "type": "FLOAT"}, + {"name": "B", "type": "FLOAT"}, + {"name": "C", "type": "STRING"}, + {"name": "D", "type": "TIMESTAMP"}, + ] + } + test_schema_2 = { + "fields": [ + {"name": "A", "type": "FLOAT"}, + {"name": "C", "type": "STRING"}, + {"name": "B", "type": "FLOAT"}, + {"name": "D", "type": "TIMESTAMP"}, + ] + } self.table.create(TABLE_ID + test_id, test_schema_1) assert self.sut.verify_schema( - self.dataset_prefix + "1", TABLE_ID + test_id, test_schema_2) + self.dataset_prefix + "1", TABLE_ID + test_id, test_schema_2 + ) def test_verify_schema_fails_different_data_type(self): test_id = "11" - test_schema_1 = {'fields': [{'name': 'A', 'type': 'FLOAT'}, - {'name': 'B', 'type': 'FLOAT'}, - {'name': 'C', 'type': 'STRING'}, - {'name': 'D', 'type': 'TIMESTAMP'}]} - test_schema_2 = {'fields': [{'name': 'A', 'type': 'FLOAT'}, - {'name': 'B', 'type': 'STRING'}, - {'name': 'C', 'type': 'STRING'}, - {'name': 'D', 'type': 'TIMESTAMP'}]} + test_schema_1 = { + "fields": [ + {"name": "A", "type": "FLOAT"}, + {"name": "B", "type": "FLOAT"}, + {"name": "C", "type": "STRING"}, + {"name": "D", "type": "TIMESTAMP"}, + ] + } + test_schema_2 = { + "fields": [ + {"name": "A", "type": "FLOAT"}, + {"name": "B", "type": "STRING"}, + {"name": "C", "type": "STRING"}, + {"name": "D", "type": "TIMESTAMP"}, + ] + } self.table.create(TABLE_ID + test_id, test_schema_1) - assert not self.sut.verify_schema(self.dataset_prefix + "1", - TABLE_ID + test_id, test_schema_2) + assert not self.sut.verify_schema( + self.dataset_prefix + "1", TABLE_ID + test_id, test_schema_2 + ) def test_verify_schema_fails_different_structure(self): test_id = "12" - test_schema_1 = {'fields': [{'name': 'A', 'type': 'FLOAT'}, - {'name': 'B', 'type': 'FLOAT'}, - {'name': 'C', 'type': 'STRING'}, - {'name': 'D', 'type': 'TIMESTAMP'}]} - test_schema_2 = {'fields': [{'name': 'A', 'type': 'FLOAT'}, - {'name': 'B2', 'type': 'FLOAT'}, - {'name': 'C', 'type': 'STRING'}, - {'name': 'D', 'type': 'TIMESTAMP'}]} + test_schema_1 = { + "fields": [ + {"name": "A", "type": "FLOAT"}, + {"name": "B", "type": "FLOAT"}, + {"name": "C", "type": "STRING"}, + {"name": "D", "type": "TIMESTAMP"}, + ] + } + test_schema_2 = { + "fields": [ + {"name": "A", "type": "FLOAT"}, + {"name": "B2", "type": "FLOAT"}, + {"name": "C", "type": "STRING"}, + {"name": "D", "type": "TIMESTAMP"}, + ] + } self.table.create(TABLE_ID + test_id, test_schema_1) assert not self.sut.verify_schema( - self.dataset_prefix + "1", TABLE_ID + test_id, test_schema_2) + self.dataset_prefix + "1", TABLE_ID + test_id, test_schema_2 + ) def test_upload_data_flexible_column_order(self, project_id): test_id = "13" @@ -1022,230 +1311,287 @@ def test_upload_data_flexible_column_order(self, project_id): df = make_mixed_dataframe_v2(test_size) # Initialize table with sample data - gbq.to_gbq(df, self.destination_table + test_id, project_id, - chunksize=10000, private_key=self.credentials) + gbq.to_gbq( + df, + self.destination_table + test_id, + project_id, + chunksize=10000, + private_key=self.credentials, + ) df_columns_reversed = df[df.columns[::-1]] - gbq.to_gbq(df_columns_reversed, self.destination_table + test_id, - project_id, if_exists='append', - private_key=self.credentials) + gbq.to_gbq( + df_columns_reversed, + self.destination_table + test_id, + project_id, + if_exists="append", + private_key=self.credentials, + ) def test_verify_schema_ignores_field_mode(self): test_id = "14" - test_schema_1 = {'fields': [{'name': 'A', - 'type': 'FLOAT', - 'mode': 'NULLABLE'}, - {'name': 'B', - 'type': 'FLOAT', - 'mode': 'NULLABLE'}, - {'name': 'C', - 'type': 'STRING', - 'mode': 'NULLABLE'}, - {'name': 'D', - 'type': 'TIMESTAMP', - 'mode': 'REQUIRED'}]} - test_schema_2 = {'fields': [{'name': 'A', - 'type': 'FLOAT'}, - {'name': 'B', - 'type': 'FLOAT'}, - {'name': 'C', - 'type': 'STRING'}, - {'name': 'D', - 'type': 'TIMESTAMP'}]} + test_schema_1 = { + "fields": [ + {"name": "A", "type": "FLOAT", "mode": "NULLABLE"}, + {"name": "B", "type": "FLOAT", "mode": "NULLABLE"}, + {"name": "C", "type": "STRING", "mode": "NULLABLE"}, + {"name": "D", "type": "TIMESTAMP", "mode": "REQUIRED"}, + ] + } + test_schema_2 = { + "fields": [ + {"name": "A", "type": "FLOAT"}, + {"name": "B", "type": "FLOAT"}, + {"name": "C", "type": "STRING"}, + {"name": "D", "type": "TIMESTAMP"}, + ] + } self.table.create(TABLE_ID + test_id, test_schema_1) assert self.sut.verify_schema( - self.dataset_prefix + "1", TABLE_ID + test_id, test_schema_2) + self.dataset_prefix + "1", TABLE_ID + test_id, test_schema_2 + ) def test_retrieve_schema(self): # Issue #24 schema function returns the schema in biquery test_id = "15" test_schema = { - 'fields': [ + "fields": [ { - 'name': 'A', - 'type': 'FLOAT', - 'mode': 'NULLABLE', - 'description': None, + "name": "A", + "type": "FLOAT", + "mode": "NULLABLE", + "description": None, }, { - 'name': 'B', - 'type': 'FLOAT', - 'mode': 'NULLABLE', - 'description': None, + "name": "B", + "type": "FLOAT", + "mode": "NULLABLE", + "description": None, }, { - 'name': 'C', - 'type': 'STRING', - 'mode': 'NULLABLE', - 'description': None, + "name": "C", + "type": "STRING", + "mode": "NULLABLE", + "description": None, }, { - 'name': 'D', - 'type': 'TIMESTAMP', - 'mode': 'NULLABLE', - 'description': None, + "name": "D", + "type": "TIMESTAMP", + "mode": "NULLABLE", + "description": None, }, ] } self.table.create(TABLE_ID + test_id, test_schema) actual = self.sut._clean_schema_fields( - self.sut.schema(self.dataset_prefix + "1", TABLE_ID + test_id)) + self.sut.schema(self.dataset_prefix + "1", TABLE_ID + test_id) + ) expected = [ - {'name': 'A', 'type': 'FLOAT'}, - {'name': 'B', 'type': 'FLOAT'}, - {'name': 'C', 'type': 'STRING'}, - {'name': 'D', 'type': 'TIMESTAMP'}, + {"name": "A", "type": "FLOAT"}, + {"name": "B", "type": "FLOAT"}, + {"name": "C", "type": "STRING"}, + {"name": "D", "type": "TIMESTAMP"}, ] - assert expected == actual, 'Expected schema used to create table' + assert expected == actual, "Expected schema used to create table" def test_schema_is_subset_passes_if_subset(self): # Issue #24 schema_is_subset indicates whether the schema of the # dataframe is a subset of the schema of the bigquery table - test_id = '16' + test_id = "16" table_name = TABLE_ID + test_id - dataset = self.dataset_prefix + '1' - - table_schema = {'fields': [{'name': 'A', - 'type': 'FLOAT'}, - {'name': 'B', - 'type': 'FLOAT'}, - {'name': 'C', - 'type': 'STRING'}]} - tested_schema = {'fields': [{'name': 'A', - 'type': 'FLOAT'}, - {'name': 'B', - 'type': 'FLOAT'}]} + dataset = self.dataset_prefix + "1" + + table_schema = { + "fields": [ + {"name": "A", "type": "FLOAT"}, + {"name": "B", "type": "FLOAT"}, + {"name": "C", "type": "STRING"}, + ] + } + tested_schema = { + "fields": [ + {"name": "A", "type": "FLOAT"}, + {"name": "B", "type": "FLOAT"}, + ] + } self.table.create(table_name, table_schema) - assert self.sut.schema_is_subset( - dataset, table_name, tested_schema) is True + assert ( + self.sut.schema_is_subset(dataset, table_name, tested_schema) + is True + ) def test_schema_is_subset_fails_if_not_subset(self): # For pull request #24 - test_id = '17' + test_id = "17" table_name = TABLE_ID + test_id - dataset = self.dataset_prefix + '1' - - table_schema = {'fields': [{'name': 'A', - 'type': 'FLOAT'}, - {'name': 'B', - 'type': 'FLOAT'}, - {'name': 'C', - 'type': 'STRING'}]} - tested_schema = {'fields': [{'name': 'A', - 'type': 'FLOAT'}, - {'name': 'C', - 'type': 'FLOAT'}]} + dataset = self.dataset_prefix + "1" + + table_schema = { + "fields": [ + {"name": "A", "type": "FLOAT"}, + {"name": "B", "type": "FLOAT"}, + {"name": "C", "type": "STRING"}, + ] + } + tested_schema = { + "fields": [ + {"name": "A", "type": "FLOAT"}, + {"name": "C", "type": "FLOAT"}, + ] + } self.table.create(table_name, table_schema) - assert self.sut.schema_is_subset( - dataset, table_name, tested_schema) is False + assert ( + self.sut.schema_is_subset(dataset, table_name, tested_schema) + is False + ) def test_upload_data_with_valid_user_schema(self, project_id): # Issue #46; tests test scenarios with user-provided # schemas df = tm.makeMixedDataFrame() test_id = "18" - test_schema = [{'name': 'A', 'type': 'FLOAT'}, - {'name': 'B', 'type': 'FLOAT'}, - {'name': 'C', 'type': 'STRING'}, - {'name': 'D', 'type': 'TIMESTAMP'}] + test_schema = [ + {"name": "A", "type": "FLOAT"}, + {"name": "B", "type": "FLOAT"}, + {"name": "C", "type": "STRING"}, + {"name": "D", "type": "TIMESTAMP"}, + ] destination_table = self.destination_table + test_id - gbq.to_gbq(df, destination_table, project_id, - private_key=self.credentials, - table_schema=test_schema) - dataset, table = destination_table.split('.') - assert self.table.verify_schema(dataset, table, - dict(fields=test_schema)) + gbq.to_gbq( + df, + destination_table, + project_id, + private_key=self.credentials, + table_schema=test_schema, + ) + dataset, table = destination_table.split(".") + assert self.table.verify_schema( + dataset, table, dict(fields=test_schema) + ) def test_upload_data_with_invalid_user_schema_raises_error( - self, project_id): + self, project_id + ): df = tm.makeMixedDataFrame() test_id = "19" - test_schema = [{'name': 'A', 'type': 'FLOAT'}, - {'name': 'B', 'type': 'FLOAT'}, - {'name': 'C', 'type': 'FLOAT'}, - {'name': 'D', 'type': 'FLOAT'}] + test_schema = [ + {"name": "A", "type": "FLOAT"}, + {"name": "B", "type": "FLOAT"}, + {"name": "C", "type": "FLOAT"}, + {"name": "D", "type": "FLOAT"}, + ] destination_table = self.destination_table + test_id with pytest.raises(gbq.GenericGBQException): - gbq.to_gbq(df, destination_table, project_id, - private_key=self.credentials, - table_schema=test_schema) + gbq.to_gbq( + df, + destination_table, + project_id, + private_key=self.credentials, + table_schema=test_schema, + ) def test_upload_data_with_missing_schema_fields_raises_error( - self, project_id): + self, project_id + ): df = tm.makeMixedDataFrame() test_id = "20" - test_schema = [{'name': 'A', 'type': 'FLOAT'}, - {'name': 'B', 'type': 'FLOAT'}, - {'name': 'C', 'type': 'FLOAT'}] + test_schema = [ + {"name": "A", "type": "FLOAT"}, + {"name": "B", "type": "FLOAT"}, + {"name": "C", "type": "FLOAT"}, + ] destination_table = self.destination_table + test_id with pytest.raises(gbq.GenericGBQException): - gbq.to_gbq(df, destination_table, project_id, - private_key=self.credentials, - table_schema=test_schema) + gbq.to_gbq( + df, + destination_table, + project_id, + private_key=self.credentials, + table_schema=test_schema, + ) def test_upload_data_with_timestamp(self, project_id): test_id = "21" test_size = 6 - df = DataFrame(np.random.randn(test_size, 4), index=range(test_size), - columns=list('ABCD')) - df['times'] = np.datetime64('2018-03-13T05:40:45.348318Z') + df = DataFrame( + np.random.randn(test_size, 4), + index=range(test_size), + columns=list("ABCD"), + ) + df["times"] = np.datetime64("2018-03-13T05:40:45.348318Z") gbq.to_gbq( - df, self.destination_table + test_id, + df, + self.destination_table + test_id, project_id=project_id, - private_key=self.credentials) + private_key=self.credentials, + ) result_df = gbq.read_gbq( - 'SELECT * FROM {0}'.format(self.destination_table + test_id), + "SELECT * FROM {0}".format(self.destination_table + test_id), project_id=project_id, private_key=self.credentials, - dialect='legacy') + dialect="legacy", + ) assert len(result_df) == test_size - expected = df['times'].sort_values() - result = result_df['times'].sort_values() + expected = df["times"].sort_values() + result = result_df["times"].sort_values() tm.assert_numpy_array_equal(expected.values, result.values) def test_upload_data_with_different_df_and_user_schema(self, project_id): df = tm.makeMixedDataFrame() - df['A'] = df['A'].astype(str) - df['B'] = df['B'].astype(str) + df["A"] = df["A"].astype(str) + df["B"] = df["B"].astype(str) test_id = "22" - test_schema = [{'name': 'A', 'type': 'FLOAT'}, - {'name': 'B', 'type': 'FLOAT'}, - {'name': 'C', 'type': 'STRING'}, - {'name': 'D', 'type': 'TIMESTAMP'}] + test_schema = [ + {"name": "A", "type": "FLOAT"}, + {"name": "B", "type": "FLOAT"}, + {"name": "C", "type": "STRING"}, + {"name": "D", "type": "TIMESTAMP"}, + ] destination_table = self.destination_table + test_id - gbq.to_gbq(df, destination_table, project_id, - private_key=self.credentials, - table_schema=test_schema) - dataset, table = destination_table.split('.') - assert self.table.verify_schema(dataset, table, - dict(fields=test_schema)) + gbq.to_gbq( + df, + destination_table, + project_id, + private_key=self.credentials, + table_schema=test_schema, + ) + dataset, table = destination_table.split(".") + assert self.table.verify_schema( + dataset, table, dict(fields=test_schema) + ) def test_upload_data_tokyo( - self, project_id, tokyo_dataset, bigquery_client): + self, project_id, tokyo_dataset, bigquery_client + ): test_size = 10 df = make_mixed_dataframe_v2(test_size) - tokyo_destination = '{}.to_gbq_test'.format(tokyo_dataset) + tokyo_destination = "{}.to_gbq_test".format(tokyo_dataset) # Initialize table with sample data gbq.to_gbq( - df, tokyo_destination, project_id, private_key=self.credentials, - location='asia-northeast1') + df, + tokyo_destination, + project_id, + private_key=self.credentials, + location="asia-northeast1", + ) table = bigquery_client.get_table( - bigquery_client.dataset(tokyo_dataset).table('to_gbq_test')) + bigquery_client.dataset(tokyo_dataset).table("to_gbq_test") + ) assert table.num_rows > 0 def test_list_dataset(self): @@ -1255,9 +1601,9 @@ def test_list_dataset(self): def test_list_table_zero_results(self, project_id): dataset_id = self.dataset_prefix + "2" self.dataset.create(dataset_id) - table_list = gbq._Dataset(project_id, - private_key=self.credentials - ).tables(dataset_id) + table_list = gbq._Dataset( + project_id, private_key=self.credentials + ).tables(dataset_id) assert len(table_list) == 0 def test_create_dataset(self): diff --git a/packages/pandas-gbq/tests/unit/test_auth.py b/packages/pandas-gbq/tests/unit/test_auth.py index 6da3f35d42fe..50c71b27615f 100644 --- a/packages/pandas-gbq/tests/unit/test_auth.py +++ b/packages/pandas-gbq/tests/unit/test_auth.py @@ -3,13 +3,13 @@ import json import os.path +from pandas_gbq import auth + try: import mock except ImportError: # pragma: NO COVER from unittest import mock -from pandas_gbq import auth - def test_get_credentials_private_key_contents(monkeypatch): from google.oauth2 import service_account @@ -23,17 +23,20 @@ def from_service_account_info(cls, key_info): monkeypatch.setattr( service_account.Credentials, - 'from_service_account_info', - from_service_account_info) - private_key = json.dumps({ - 'private_key': 'some_key', - 'client_email': 'service-account@example.com', - 'project_id': 'private-key-project' - }) + "from_service_account_info", + from_service_account_info, + ) + private_key = json.dumps( + { + "private_key": "some_key", + "client_email": "service-account@example.com", + "project_id": "private-key-project", + } + ) credentials, project = auth.get_credentials(private_key=private_key) assert credentials is not None - assert project == 'private-key-project' + assert project == "private-key-project" def test_get_credentials_private_key_path(monkeypatch): @@ -48,10 +51,12 @@ def from_service_account_info(cls, key_info): monkeypatch.setattr( service_account.Credentials, - 'from_service_account_info', - from_service_account_info) + "from_service_account_info", + from_service_account_info, + ) private_key = os.path.join( - os.path.dirname(__file__), '..', 'data', 'dummy_key.json') + os.path.dirname(__file__), "..", "data", "dummy_key.json" + ) credentials, project = auth.get_credentials(private_key=private_key) assert credentials is not None @@ -66,15 +71,15 @@ def test_get_credentials_default_credentials(monkeypatch): def mock_default_credentials(scopes=None, request=None): return ( mock.create_autospec(google.auth.credentials.Credentials), - 'default-project', + "default-project", ) - monkeypatch.setattr(google.auth, 'default', mock_default_credentials) + monkeypatch.setattr(google.auth, "default", mock_default_credentials) mock_client = mock.create_autospec(google.cloud.bigquery.Client) - monkeypatch.setattr(google.cloud.bigquery, 'Client', mock_client) + monkeypatch.setattr(google.cloud.bigquery, "Client", mock_client) credentials, project = auth.get_credentials() - assert project == 'default-project' + assert project == "default-project" assert credentials is not None @@ -85,17 +90,17 @@ def test_get_credentials_load_user_no_default(monkeypatch): def mock_default_credentials(scopes=None, request=None): return (None, None) - monkeypatch.setattr(google.auth, 'default', mock_default_credentials) + monkeypatch.setattr(google.auth, "default", mock_default_credentials) mock_user_credentials = mock.create_autospec( - google.auth.credentials.Credentials) + google.auth.credentials.Credentials + ) def mock_load_credentials(project_id=None, credentials_path=None): return mock_user_credentials monkeypatch.setattr( - auth, - 'load_user_account_credentials', - mock_load_credentials) + auth, "load_user_account_credentials", mock_load_credentials + ) credentials, project = auth.get_credentials() assert project is None diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index 216d1451505a..df80d5d28e5f 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -14,13 +14,15 @@ from unittest import mock pytestmark = pytest.mark.filter_warnings( - "ignore:credentials from Google Cloud SDK") + "ignore:credentials from Google Cloud SDK" +) @pytest.fixture def min_bq_version(): import pkg_resources - return pkg_resources.parse_version('0.32.0') + + return pkg_resources.parse_version("0.32.0") @pytest.fixture(autouse=True) @@ -28,25 +30,22 @@ def mock_bigquery_client(monkeypatch): from google.api_core.exceptions import NotFound import google.cloud.bigquery import google.cloud.bigquery.table + mock_client = mock.create_autospec(google.cloud.bigquery.Client) - mock_schema = [ - google.cloud.bigquery.SchemaField('_f0', 'INTEGER') - ] + mock_schema = [google.cloud.bigquery.SchemaField("_f0", "INTEGER")] # Mock out SELECT 1 query results. mock_query = mock.create_autospec(google.cloud.bigquery.QueryJob) - mock_query.job_id = 'some-random-id' - mock_query.state = 'DONE' - mock_rows = mock.create_autospec( - google.cloud.bigquery.table.RowIterator) + mock_query.job_id = "some-random-id" + mock_query.state = "DONE" + mock_rows = mock.create_autospec(google.cloud.bigquery.table.RowIterator) mock_rows.total_rows = 1 mock_rows.schema = mock_schema mock_rows.__iter__.return_value = [(1,)] mock_query.result.return_value = mock_rows mock_client.query.return_value = mock_query # Mock table creation. - mock_client.get_table.side_effect = NotFound('nope') - monkeypatch.setattr( - gbq.GbqConnector, 'get_client', lambda _: mock_client) + mock_client.get_table.side_effect = NotFound("nope") + monkeypatch.setattr(gbq.GbqConnector, "get_client", lambda _: mock_client) def mock_none_credentials(*args, **kwargs): @@ -55,115 +54,136 @@ def mock_none_credentials(*args, **kwargs): def mock_get_credentials(*args, **kwargs): import google.auth.credentials + mock_credentials = mock.create_autospec( - google.auth.credentials.Credentials) - return mock_credentials, 'default-project' + google.auth.credentials.Credentials + ) + return mock_credentials, "default-project" def mock_get_user_credentials(*args, **kwargs): import google.auth.credentials + mock_credentials = mock.create_autospec( - google.auth.credentials.Credentials) + google.auth.credentials.Credentials + ) return mock_credentials @pytest.fixture(autouse=True) def no_auth(monkeypatch): from pandas_gbq import auth + monkeypatch.setattr( - auth, 'get_application_default_credentials', mock_get_credentials) + auth, "get_application_default_credentials", mock_get_credentials + ) monkeypatch.setattr( - auth, 'get_user_account_credentials', mock_get_user_credentials) + auth, "get_user_account_credentials", mock_get_user_credentials + ) monkeypatch.setattr( - auth, '_try_credentials', lambda project_id, credentials: credentials) + auth, "_try_credentials", lambda project_id, credentials: credentials + ) def test_should_return_credentials_path_set_by_env_var(): - env = {'PANDAS_GBQ_CREDENTIALS_FILE': '/tmp/dummy.dat'} - with mock.patch.dict('os.environ', env): - assert gbq._get_credentials_file() == '/tmp/dummy.dat' + env = {"PANDAS_GBQ_CREDENTIALS_FILE": "/tmp/dummy.dat"} + with mock.patch.dict("os.environ", env): + assert gbq._get_credentials_file() == "/tmp/dummy.dat" @pytest.mark.parametrize( - ('input', 'type_', 'expected'), [ - (1, 'INTEGER', int(1)), - (1, 'FLOAT', float(1)), - pytest.param('false', 'BOOLEAN', False, marks=pytest.mark.xfail), + ("input", "type_", "expected"), + [ + (1, "INTEGER", int(1)), + (1, "FLOAT", float(1)), + pytest.param("false", "BOOLEAN", False, marks=pytest.mark.xfail), pytest.param( - '0e9', 'TIMESTAMP', - np_datetime64_compat('1970-01-01T00:00:00Z'), - marks=pytest.mark.xfail), - ('STRING', 'STRING', 'STRING'), - ]) -def test_should_return_bigquery_correctly_typed( - input, type_, expected): + "0e9", + "TIMESTAMP", + np_datetime64_compat("1970-01-01T00:00:00Z"), + marks=pytest.mark.xfail, + ), + ("STRING", "STRING", "STRING"), + ], +) +def test_should_return_bigquery_correctly_typed(input, type_, expected): result = gbq._parse_data( - dict(fields=[dict(name='x', type=type_, mode='NULLABLE')]), - rows=[[input]]).iloc[0, 0] + dict(fields=[dict(name="x", type=type_, mode="NULLABLE")]), + rows=[[input]], + ).iloc[0, 0] assert result == expected def test_to_gbq_should_fail_if_invalid_table_name_passed(): with pytest.raises(gbq.NotFoundException): - gbq.to_gbq(DataFrame([[1]]), 'invalid_table_name', project_id="1234") + gbq.to_gbq(DataFrame([[1]]), "invalid_table_name", project_id="1234") def test_to_gbq_with_no_project_id_given_should_fail(monkeypatch): from pandas_gbq import auth + monkeypatch.setattr( - auth, 'get_application_default_credentials', mock_none_credentials) + auth, "get_application_default_credentials", mock_none_credentials + ) with pytest.raises(ValueError) as exception: - gbq.to_gbq(DataFrame([[1]]), 'dataset.tablename') - assert 'Could not determine project ID' in str(exception) + gbq.to_gbq(DataFrame([[1]]), "dataset.tablename") + assert "Could not determine project ID" in str(exception) def test_to_gbq_with_verbose_new_pandas_warns_deprecation(min_bq_version): import pkg_resources - pandas_version = pkg_resources.parse_version('0.23.0') - with pytest.warns(FutureWarning), \ - mock.patch( - 'pkg_resources.Distribution.parsed_version', - new_callable=mock.PropertyMock) as mock_version: + + pandas_version = pkg_resources.parse_version("0.23.0") + with pytest.warns(FutureWarning), mock.patch( + "pkg_resources.Distribution.parsed_version", + new_callable=mock.PropertyMock, + ) as mock_version: mock_version.side_effect = [min_bq_version, pandas_version] try: gbq.to_gbq( DataFrame([[1]]), - 'dataset.tablename', - project_id='my-project', - verbose=True) + "dataset.tablename", + project_id="my-project", + verbose=True, + ) except gbq.TableCreationError: pass def test_to_gbq_with_not_verbose_new_pandas_warns_deprecation(min_bq_version): import pkg_resources - pandas_version = pkg_resources.parse_version('0.23.0') - with pytest.warns(FutureWarning), \ - mock.patch( - 'pkg_resources.Distribution.parsed_version', - new_callable=mock.PropertyMock) as mock_version: + + pandas_version = pkg_resources.parse_version("0.23.0") + with pytest.warns(FutureWarning), mock.patch( + "pkg_resources.Distribution.parsed_version", + new_callable=mock.PropertyMock, + ) as mock_version: mock_version.side_effect = [min_bq_version, pandas_version] try: gbq.to_gbq( DataFrame([[1]]), - 'dataset.tablename', - project_id='my-project', - verbose=False) + "dataset.tablename", + project_id="my-project", + verbose=False, + ) except gbq.TableCreationError: pass def test_to_gbq_wo_verbose_w_new_pandas_no_warnings(recwarn, min_bq_version): import pkg_resources - pandas_version = pkg_resources.parse_version('0.23.0') + + pandas_version = pkg_resources.parse_version("0.23.0") with mock.patch( - 'pkg_resources.Distribution.parsed_version', - new_callable=mock.PropertyMock) as mock_version: + "pkg_resources.Distribution.parsed_version", + new_callable=mock.PropertyMock, + ) as mock_version: mock_version.side_effect = [min_bq_version, pandas_version] try: gbq.to_gbq( - DataFrame([[1]]), 'dataset.tablename', project_id='my-project') + DataFrame([[1]]), "dataset.tablename", project_id="my-project" + ) except gbq.TableCreationError: pass assert len(recwarn) == 0 @@ -171,17 +191,20 @@ def test_to_gbq_wo_verbose_w_new_pandas_no_warnings(recwarn, min_bq_version): def test_to_gbq_with_verbose_old_pandas_no_warnings(recwarn, min_bq_version): import pkg_resources - pandas_version = pkg_resources.parse_version('0.22.0') + + pandas_version = pkg_resources.parse_version("0.22.0") with mock.patch( - 'pkg_resources.Distribution.parsed_version', - new_callable=mock.PropertyMock) as mock_version: + "pkg_resources.Distribution.parsed_version", + new_callable=mock.PropertyMock, + ) as mock_version: mock_version.side_effect = [min_bq_version, pandas_version] try: gbq.to_gbq( DataFrame([[1]]), - 'dataset.tablename', - project_id='my-project', - verbose=True) + "dataset.tablename", + project_id="my-project", + verbose=True, + ) except gbq.TableCreationError: pass assert len(recwarn) == 0 @@ -189,125 +212,151 @@ def test_to_gbq_with_verbose_old_pandas_no_warnings(recwarn, min_bq_version): def test_read_gbq_with_no_project_id_given_should_fail(monkeypatch): from pandas_gbq import auth + monkeypatch.setattr( - auth, 'get_application_default_credentials', mock_none_credentials) + auth, "get_application_default_credentials", mock_none_credentials + ) with pytest.raises(ValueError) as exception: - gbq.read_gbq('SELECT 1', dialect='standard') - assert 'Could not determine project ID' in str(exception) + gbq.read_gbq("SELECT 1", dialect="standard") + assert "Could not determine project ID" in str(exception) def test_read_gbq_with_inferred_project_id(monkeypatch): - df = gbq.read_gbq('SELECT 1', dialect='standard') + df = gbq.read_gbq("SELECT 1", dialect="standard") assert df is not None def test_that_parse_data_works_properly(): from google.cloud.bigquery.table import Row - test_schema = {'fields': [ - {'mode': 'NULLABLE', 'name': 'column_x', 'type': 'STRING'}]} - field_to_index = {'column_x': 0} - values = ('row_value',) + + test_schema = { + "fields": [{"mode": "NULLABLE", "name": "column_x", "type": "STRING"}] + } + field_to_index = {"column_x": 0} + values = ("row_value",) test_page = [Row(values, field_to_index)] test_output = gbq._parse_data(test_schema, test_page) - correct_output = DataFrame({'column_x': ['row_value']}) + correct_output = DataFrame({"column_x": ["row_value"]}) tm.assert_frame_equal(test_output, correct_output) def test_read_gbq_with_invalid_private_key_json_should_fail(): with pytest.raises(pandas_gbq.exceptions.InvalidPrivateKeyFormat): gbq.read_gbq( - 'SELECT 1', dialect='standard', project_id='x', private_key='y') + "SELECT 1", dialect="standard", project_id="x", private_key="y" + ) def test_read_gbq_with_empty_private_key_json_should_fail(): with pytest.raises(pandas_gbq.exceptions.InvalidPrivateKeyFormat): gbq.read_gbq( - 'SELECT 1', dialect='standard', project_id='x', private_key='{}') + "SELECT 1", dialect="standard", project_id="x", private_key="{}" + ) def test_read_gbq_with_private_key_json_wrong_types_should_fail(): with pytest.raises(pandas_gbq.exceptions.InvalidPrivateKeyFormat): gbq.read_gbq( - 'SELECT 1', dialect='standard', project_id='x', - private_key='{ "client_email" : 1, "private_key" : True }') + "SELECT 1", + dialect="standard", + project_id="x", + private_key='{ "client_email" : 1, "private_key" : True }', + ) def test_read_gbq_with_empty_private_key_file_should_fail(): with tm.ensure_clean() as empty_file_path: with pytest.raises(pandas_gbq.exceptions.InvalidPrivateKeyFormat): - gbq.read_gbq('SELECT 1', dialect='standard', project_id='x', - private_key=empty_file_path) + gbq.read_gbq( + "SELECT 1", + dialect="standard", + project_id="x", + private_key=empty_file_path, + ) def test_read_gbq_with_corrupted_private_key_json_should_fail(): with pytest.raises(pandas_gbq.exceptions.InvalidPrivateKeyFormat): gbq.read_gbq( - 'SELECT 1', dialect='standard', project_id='x', - private_key='99999999999999999') + "SELECT 1", + dialect="standard", + project_id="x", + private_key="99999999999999999", + ) def test_read_gbq_with_verbose_new_pandas_warns_deprecation(min_bq_version): import pkg_resources - pandas_version = pkg_resources.parse_version('0.23.0') - with pytest.warns(FutureWarning), \ - mock.patch( - 'pkg_resources.Distribution.parsed_version', - new_callable=mock.PropertyMock) as mock_version: + + pandas_version = pkg_resources.parse_version("0.23.0") + with pytest.warns(FutureWarning), mock.patch( + "pkg_resources.Distribution.parsed_version", + new_callable=mock.PropertyMock, + ) as mock_version: mock_version.side_effect = [min_bq_version, pandas_version] - gbq.read_gbq('SELECT 1', project_id='my-project', verbose=True) + gbq.read_gbq("SELECT 1", project_id="my-project", verbose=True) def test_read_gbq_with_not_verbose_new_pandas_warns_deprecation( - min_bq_version): + min_bq_version +): import pkg_resources - pandas_version = pkg_resources.parse_version('0.23.0') - with pytest.warns(FutureWarning), \ - mock.patch( - 'pkg_resources.Distribution.parsed_version', - new_callable=mock.PropertyMock) as mock_version: + + pandas_version = pkg_resources.parse_version("0.23.0") + with pytest.warns(FutureWarning), mock.patch( + "pkg_resources.Distribution.parsed_version", + new_callable=mock.PropertyMock, + ) as mock_version: mock_version.side_effect = [min_bq_version, pandas_version] - gbq.read_gbq('SELECT 1', project_id='my-project', verbose=False) + gbq.read_gbq("SELECT 1", project_id="my-project", verbose=False) def test_read_gbq_wo_verbose_w_new_pandas_no_warnings(recwarn, min_bq_version): import pkg_resources - pandas_version = pkg_resources.parse_version('0.23.0') + + pandas_version = pkg_resources.parse_version("0.23.0") with mock.patch( - 'pkg_resources.Distribution.parsed_version', - new_callable=mock.PropertyMock) as mock_version: + "pkg_resources.Distribution.parsed_version", + new_callable=mock.PropertyMock, + ) as mock_version: mock_version.side_effect = [min_bq_version, pandas_version] - gbq.read_gbq('SELECT 1', project_id='my-project', dialect='standard') + gbq.read_gbq("SELECT 1", project_id="my-project", dialect="standard") assert len(recwarn) == 0 def test_read_gbq_with_verbose_old_pandas_no_warnings(recwarn, min_bq_version): import pkg_resources - pandas_version = pkg_resources.parse_version('0.22.0') + + pandas_version = pkg_resources.parse_version("0.22.0") with mock.patch( - 'pkg_resources.Distribution.parsed_version', - new_callable=mock.PropertyMock) as mock_version: + "pkg_resources.Distribution.parsed_version", + new_callable=mock.PropertyMock, + ) as mock_version: mock_version.side_effect = [min_bq_version, pandas_version] gbq.read_gbq( - 'SELECT 1', project_id='my-project', dialect='standard', - verbose=True) + "SELECT 1", + project_id="my-project", + dialect="standard", + verbose=True, + ) assert len(recwarn) == 0 def test_read_gbq_with_invalid_dialect(): with pytest.raises(ValueError) as excinfo: - gbq.read_gbq('SELECT 1', dialect='invalid') - assert 'is not valid for dialect' in str(excinfo.value) + gbq.read_gbq("SELECT 1", dialect="invalid") + assert "is not valid for dialect" in str(excinfo.value) def test_read_gbq_without_dialect_warns_future_change(): with pytest.warns(FutureWarning): - gbq.read_gbq('SELECT 1') + gbq.read_gbq("SELECT 1") def test_generate_bq_schema_deprecated(): # 11121 Deprecation of generate_bq_schema with pytest.warns(FutureWarning): - df = DataFrame([[1, 'two'], [3, 'four']]) + df = DataFrame([[1, "two"], [3, "four"]]) gbq.generate_bq_schema(df) diff --git a/packages/pandas-gbq/tests/unit/test_load.py b/packages/pandas-gbq/tests/unit/test_load.py index fdbedc4607fc..b53a4dc82203 100644 --- a/packages/pandas-gbq/tests/unit/test_load.py +++ b/packages/pandas-gbq/tests/unit/test_load.py @@ -1,10 +1,11 @@ # -*- coding: utf-8 -*- +from io import StringIO + import numpy import pandas from pandas_gbq import load -from io import StringIO def test_encode_chunk_with_unicode(): @@ -13,12 +14,13 @@ def test_encode_chunk_with_unicode(): See: https://github.com/pydata/pandas-gbq/issues/106 """ df = pandas.DataFrame( - numpy.random.randn(6, 4), index=range(6), columns=list('ABCD')) - df['s'] = u'信用卡' + numpy.random.randn(6, 4), index=range(6), columns=list("ABCD") + ) + df["s"] = u"信用卡" csv_buffer = load.encode_chunk(df) csv_bytes = csv_buffer.read() - csv_string = csv_bytes.decode('utf-8') - assert u'信用卡' in csv_string + csv_string = csv_bytes.decode("utf-8") + assert u"信用卡" in csv_string def test_encode_chunk_with_floats(): @@ -27,12 +29,12 @@ def test_encode_chunk_with_floats(): See: https://github.com/pydata/pandas-gbq/issues/192 """ - input_csv = StringIO(u'01/01/17 23:00,1.05148,1.05153,1.05148,1.05153,4') + input_csv = StringIO(u"01/01/17 23:00,1.05148,1.05153,1.05148,1.05153,4") df = pandas.read_csv(input_csv, header=None) csv_buffer = load.encode_chunk(df) csv_bytes = csv_buffer.read() - csv_string = csv_bytes.decode('utf-8') - assert '1.05153' in csv_string + csv_string = csv_bytes.decode("utf-8") + assert "1.05153" in csv_string def test_encode_chunks_splits_dataframe(): diff --git a/packages/pandas-gbq/tests/unit/test_schema.py b/packages/pandas-gbq/tests/unit/test_schema.py index 4d6b0add3958..66aca1dc869b 100644 --- a/packages/pandas-gbq/tests/unit/test_schema.py +++ b/packages/pandas-gbq/tests/unit/test_schema.py @@ -8,48 +8,50 @@ @pytest.mark.parametrize( - 'dataframe,expected_schema', + "dataframe,expected_schema", [ ( - pandas.DataFrame(data={'col1': [1, 2, 3]}), - {'fields': [{'name': 'col1', 'type': 'INTEGER'}]}, + pandas.DataFrame(data={"col1": [1, 2, 3]}), + {"fields": [{"name": "col1", "type": "INTEGER"}]}, ), ( - pandas.DataFrame(data={'col1': [True, False]}), - {'fields': [{'name': 'col1', 'type': 'BOOLEAN'}]}, + pandas.DataFrame(data={"col1": [True, False]}), + {"fields": [{"name": "col1", "type": "BOOLEAN"}]}, ), ( - pandas.DataFrame(data={'col1': [1.0, 3.14]}), - {'fields': [{'name': 'col1', 'type': 'FLOAT'}]}, + pandas.DataFrame(data={"col1": [1.0, 3.14]}), + {"fields": [{"name": "col1", "type": "FLOAT"}]}, ), ( - pandas.DataFrame(data={'col1': [u'hello', u'world']}), - {'fields': [{'name': 'col1', 'type': 'STRING'}]}, + pandas.DataFrame(data={"col1": [u"hello", u"world"]}), + {"fields": [{"name": "col1", "type": "STRING"}]}, ), ( - pandas.DataFrame(data={'col1': [datetime.datetime.now()]}), - {'fields': [{'name': 'col1', 'type': 'TIMESTAMP'}]}, + pandas.DataFrame(data={"col1": [datetime.datetime.now()]}), + {"fields": [{"name": "col1", "type": "TIMESTAMP"}]}, ), ( pandas.DataFrame( data={ - 'col1': [datetime.datetime.now()], - 'col2': [u'hello'], - 'col3': [3.14], - 'col4': [True], - 'col5': [4], - }), + "col1": [datetime.datetime.now()], + "col2": [u"hello"], + "col3": [3.14], + "col4": [True], + "col5": [4], + } + ), { - 'fields': [ - {'name': 'col1', 'type': 'TIMESTAMP'}, - {'name': 'col2', 'type': 'STRING'}, - {'name': 'col3', 'type': 'FLOAT'}, - {'name': 'col4', 'type': 'BOOLEAN'}, - {'name': 'col5', 'type': 'INTEGER'}, - ], + "fields": [ + {"name": "col1", "type": "TIMESTAMP"}, + {"name": "col2", "type": "STRING"}, + {"name": "col3", "type": "FLOAT"}, + {"name": "col4", "type": "BOOLEAN"}, + {"name": "col5", "type": "INTEGER"}, + ] }, ), - ]) + ], +) def test_generate_bq_schema(dataframe, expected_schema): schema = pandas_gbq.schema.generate_bq_schema(dataframe) assert schema == expected_schema From ebe80de0d8c432ede65e35209eaf0cead4f2f033 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 31 Aug 2018 13:49:25 -0700 Subject: [PATCH 144/519] [BUG] skip _try_credentials check for to_gbq (#207) * [BUG] skip _try_credentials check for to_gbq Don't do a query read when all you need to write is write credentials. * Fix auth tests. * Fix mock for Python 2.7. --- packages/pandas-gbq/docs/source/changelog.rst | 1 + packages/pandas-gbq/pandas_gbq/auth.py | 27 ++++++++++++++----- packages/pandas-gbq/pandas_gbq/gbq.py | 5 ++++ packages/pandas-gbq/tests/system/test_auth.py | 25 ++++++++++++----- packages/pandas-gbq/tests/unit/test_auth.py | 4 ++- packages/pandas-gbq/tests/unit/test_gbq.py | 14 ++++++++++ 6 files changed, 62 insertions(+), 14 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index b4b534885ea7..8681de2723c8 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -12,6 +12,7 @@ Changelog (:issue:`128`) - Reduced verbosity of logging from ``read_gbq``, particularly for short queries. (:issue:`201`) +- Avoid ``SELECT 1`` query when running ``to_gbq``. (:issue:`202`) .. _changelog-0.6.0: diff --git a/packages/pandas-gbq/pandas_gbq/auth.py b/packages/pandas-gbq/pandas_gbq/auth.py index c27d342e6aeb..2bc2efeac971 100644 --- a/packages/pandas-gbq/pandas_gbq/auth.py +++ b/packages/pandas-gbq/pandas_gbq/auth.py @@ -16,20 +16,28 @@ def get_credentials( - private_key=None, project_id=None, reauth=False, auth_local_webserver=False + private_key=None, + project_id=None, + reauth=False, + auth_local_webserver=False, + try_credentials=None, ): + if try_credentials is None: + try_credentials = _try_credentials + if private_key: return get_service_account_credentials(private_key) # Try to retrieve Application Default Credentials credentials, default_project = get_application_default_credentials( - project_id=project_id + try_credentials, project_id=project_id ) if credentials: return credentials, default_project credentials = get_user_account_credentials( + try_credentials, project_id=project_id, reauth=reauth, auth_local_webserver=auth_local_webserver, @@ -79,7 +87,7 @@ def get_service_account_credentials(private_key): ) -def get_application_default_credentials(project_id=None): +def get_application_default_credentials(try_credentials, project_id=None): """ This method tries to retrieve the "default application credentials". This could be useful for running code on Google Cloud Platform. @@ -111,10 +119,11 @@ def get_application_default_credentials(project_id=None): # used with BigQuery. For example, we could be running on a GCE instance # that does not allow the BigQuery scopes. billing_project = project_id or default_project - return _try_credentials(billing_project, credentials), billing_project + return try_credentials(billing_project, credentials), billing_project def get_user_account_credentials( + try_credentials, project_id=None, reauth=False, auth_local_webserver=False, @@ -151,7 +160,9 @@ def get_user_account_credentials( os.rename("bigquery_credentials.dat", credentials_path) credentials = load_user_account_credentials( - project_id=project_id, credentials_path=credentials_path + try_credentials, + project_id=project_id, + credentials_path=credentials_path, ) client_config = { @@ -187,7 +198,9 @@ def get_user_account_credentials( return credentials -def load_user_account_credentials(project_id=None, credentials_path=None): +def load_user_account_credentials( + try_credentials, project_id=None, credentials_path=None +): """ Loads user account credentials from a local file. @@ -230,7 +243,7 @@ def load_user_account_credentials(project_id=None, credentials_path=None): request = google.auth.transport.requests.Request() credentials.refresh(request) - return _try_credentials(project_id, credentials) + return try_credentials(project_id, credentials) def get_default_credentials_path(): diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index eba08009c800..c45384e45858 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -171,6 +171,7 @@ def __init__( auth_local_webserver=False, dialect="legacy", location=None, + try_credentials=None, ): from google.api_core.exceptions import GoogleAPIError from google.api_core.exceptions import ClientError @@ -189,6 +190,7 @@ def __init__( project_id=project_id, reauth=reauth, auth_local_webserver=auth_local_webserver, + try_credentials=try_credentials, ) if self.project_id is None: @@ -804,6 +806,9 @@ def to_gbq( private_key=private_key, auth_local_webserver=auth_local_webserver, location=location, + # Avoid reads when writing tables. + # https://github.com/pydata/pandas-gbq/issues/202 + try_credentials=lambda project, creds: creds, ) dataset_id, table_id = destination_table.rsplit(".", 1) diff --git a/packages/pandas-gbq/tests/system/test_auth.py b/packages/pandas-gbq/tests/system/test_auth.py index 7ccc79c22fbc..f7cdf014769d 100644 --- a/packages/pandas-gbq/tests/system/test_auth.py +++ b/packages/pandas-gbq/tests/system/test_auth.py @@ -66,9 +66,13 @@ def test_get_application_default_credentials_does_not_throw_error(): with mock.patch( "google.auth.default", side_effect=DefaultCredentialsError() ): - credentials, _ = auth.get_application_default_credentials() + credentials, _ = auth.get_application_default_credentials( + try_credentials=auth._try_credentials + ) else: - credentials, _ = auth.get_application_default_credentials() + credentials, _ = auth.get_application_default_credentials( + try_credentials=auth._try_credentials + ) assert credentials is None @@ -77,7 +81,9 @@ def test_get_application_default_credentials_returns_credentials(): pytest.skip("Cannot get default_credentials " "from the environment!") from google.auth.credentials import Credentials - credentials, default_project = auth.get_application_default_credentials() + credentials, default_project = auth.get_application_default_credentials( + try_credentials=auth._try_credentials + ) assert isinstance(credentials, Credentials) assert default_project is not None @@ -88,7 +94,9 @@ def test_get_user_account_credentials_bad_file_returns_credentials(): from google.auth.credentials import Credentials with mock.patch("__main__.open", side_effect=IOError()): - credentials = auth.get_user_account_credentials() + credentials = auth.get_user_account_credentials( + try_credentials=auth._try_credentials + ) assert isinstance(credentials, Credentials) @@ -97,7 +105,9 @@ def test_get_user_account_credentials_returns_credentials(project_id): from google.auth.credentials import Credentials credentials = auth.get_user_account_credentials( - project_id=project_id, auth_local_webserver=True + project_id=project_id, + auth_local_webserver=True, + try_credentials=auth._try_credentials, ) assert isinstance(credentials, Credentials) @@ -107,6 +117,9 @@ def test_get_user_account_credentials_reauth_returns_credentials(project_id): from google.auth.credentials import Credentials credentials = auth.get_user_account_credentials( - project_id=project_id, auth_local_webserver=True, reauth=True + project_id=project_id, + auth_local_webserver=True, + reauth=True, + try_credentials=auth._try_credentials, ) assert isinstance(credentials, Credentials) diff --git a/packages/pandas-gbq/tests/unit/test_auth.py b/packages/pandas-gbq/tests/unit/test_auth.py index 50c71b27615f..d8107a40b78b 100644 --- a/packages/pandas-gbq/tests/unit/test_auth.py +++ b/packages/pandas-gbq/tests/unit/test_auth.py @@ -95,7 +95,9 @@ def mock_default_credentials(scopes=None, request=None): google.auth.credentials.Credentials ) - def mock_load_credentials(project_id=None, credentials_path=None): + def mock_load_credentials( + try_credentials, project_id=None, credentials_path=None + ): return mock_user_credentials monkeypatch.setattr( diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index df80d5d28e5f..4a42e057f78b 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -46,6 +46,7 @@ def mock_bigquery_client(monkeypatch): # Mock table creation. mock_client.get_table.side_effect = NotFound("nope") monkeypatch.setattr(gbq.GbqConnector, "get_client", lambda _: mock_client) + return mock_client def mock_none_credentials(*args, **kwargs): @@ -210,6 +211,19 @@ def test_to_gbq_with_verbose_old_pandas_no_warnings(recwarn, min_bq_version): assert len(recwarn) == 0 +def test_to_gbq_doesnt_run_query( + recwarn, mock_bigquery_client, min_bq_version +): + try: + gbq.to_gbq( + DataFrame([[1]]), "dataset.tablename", project_id="my-project" + ) + except gbq.TableCreationError: + pass + + mock_bigquery_client.query.assert_not_called() + + def test_read_gbq_with_no_project_id_given_should_fail(monkeypatch): from pandas_gbq import auth From 2fbfbc460adeed67dda4e08f603defa8edc6fa4e Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Fri, 31 Aug 2018 23:21:02 -0400 Subject: [PATCH 145/519] [TST] skip conda test (#209) --- packages/pandas-gbq/.travis.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/pandas-gbq/.travis.yml b/packages/pandas-gbq/.travis.yml index b1ab8f9c3d14..a9546b88b77a 100644 --- a/packages/pandas-gbq/.travis.yml +++ b/packages/pandas-gbq/.travis.yml @@ -8,9 +8,10 @@ matrix: - os: linux python: 3.5 env: PYTHON=3.5 PANDAS=0.18.1 COVERAGE='true' LINT='false' - - os: linux - python: 3.6 - env: PYTHON=3.6 PANDAS=0.20.1 COVERAGE='false' LINT='false' + # https://github.com/pydata/pandas-gbq/issues/189 + # - os: linux + # python: 3.6 + # env: PYTHON=3.6 PANDAS=0.20.1 COVERAGE='false' LINT='false' - os: linux python: 3.6 env: PYTHON=3.6 PANDAS=MASTER COVERAGE='false' LINT='true' From 3d98b0f97293abb0435ac3d29f8dada89bd84846 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 4 Sep 2018 09:19:44 -0700 Subject: [PATCH 146/519] Prepare for 0.6.1 release. (#210) --- packages/pandas-gbq/docs/source/changelog.rst | 2 +- packages/pandas-gbq/release-procedure.md | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 8681de2723c8..071ad529b05b 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -3,7 +3,7 @@ Changelog .. _changelog-0.6.1: -0.6.1 / [unreleased] +0.6.1 / 2018-09-11 -------------------- - Improved ``read_gbq`` performance and memory consumption by delegating diff --git a/packages/pandas-gbq/release-procedure.md b/packages/pandas-gbq/release-procedure.md index 3b3384b98515..e9a1615bc4e5 100644 --- a/packages/pandas-gbq/release-procedure.md +++ b/packages/pandas-gbq/release-procedure.md @@ -1,3 +1,14 @@ +* Add current date to `docs/source/changelog.rst`. + +* Send PR to prepare release on scheduled date. + +* Verify you are on the latest changes. `rebase -i` should be noop. + + git fetch pandas-gbq master + git checkout master + git rebase -i pandas-gbq/master + git diff pandas-gbq/master + * Tag commit git tag -a x.x.x -m 'Version x.x.x' From a7e76597a6ae75fefcd440e63aa49c448818daaa Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 4 Sep 2018 09:32:01 -0700 Subject: [PATCH 147/519] Add more details to release steps. --- packages/pandas-gbq/release-procedure.md | 32 +++++++++++++++--------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/packages/pandas-gbq/release-procedure.md b/packages/pandas-gbq/release-procedure.md index e9a1615bc4e5..58dfda9b59dd 100644 --- a/packages/pandas-gbq/release-procedure.md +++ b/packages/pandas-gbq/release-procedure.md @@ -1,8 +1,9 @@ -* Add current date to `docs/source/changelog.rst`. * Send PR to prepare release on scheduled date. -* Verify you are on the latest changes. `rebase -i` should be noop. + * Add current date and any missing changes to [`docs/source/changelog.rst`](https://github.com/pydata/pandas-gbq/blob/master/docs/source/changelog.rst). + +* Verify your local repository is on the latest changes. `rebase -i` should be noop. git fetch pandas-gbq master git checkout master @@ -13,28 +14,35 @@ git tag -a x.x.x -m 'Version x.x.x' -* and push to github +* Push to GitHub git push pandas-gbq master --tags -* Build the package +* Build the package git clean -xfd python setup.py register sdist bdist_wheel --universal -* Upload to test PyPI +* Upload to test PyPI - twine upload --repository testpypi dist/* + twine upload --repository testpypi dist/* -* Try out test PyPI package +* Try out test PyPI package - pip install --upgrade --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple pandas-gbq + pip install --upgrade --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple pandas-gbq -* Upload to PyPI +* Upload to PyPI twine upload dist/* -* Do a pull-request to the feedstock on `pandas-gbq-feedstock `__ +* Create the [release on GitHub](https://github.com/pydata/pandas-gbq/releases/new) using the tag created earlier. + + * Copy release notes from [changelog.rst](https://github.com/pydata/pandas-gbq/blob/master/docs/source/changelog.rst). + * Upload wheel and source zip from `dist/` directory. + +* Do a pull-request to the feedstock on `pandas-gbq-feedstock `__ + (Or review PR from @regro-cf-autotick-bot which updates the feedstock). - update the version - update the SHA256 (retrieve from PyPI) + * update the version + * update the SHA256 (retrieve from PyPI) + * update the dependencies (if they changed) From 92a758e39fb37fa6b468b8fc6b033178107a5516 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 4 Sep 2018 14:21:28 -0700 Subject: [PATCH 148/519] [ENH] cache credentials in-memory (#208) * [ENH] cache credentials in-memory This commit adds a global pandas_gbq.context variable which caches the project ID and credentials across calls to read_gbq and to_gbq. * Add Context to changelog. * Fix for Python 3.5 --- packages/pandas-gbq/docs/source/api.rst | 4 + packages/pandas-gbq/docs/source/changelog.rst | 8 ++ packages/pandas-gbq/pandas_gbq/__init__.py | 2 +- packages/pandas-gbq/pandas_gbq/gbq.py | 94 +++++++++++++++++-- packages/pandas-gbq/tests/unit/conftest.py | 41 ++++++++ .../pandas-gbq/tests/unit/test_context.py | 38 ++++++++ packages/pandas-gbq/tests/unit/test_gbq.py | 24 ----- 7 files changed, 179 insertions(+), 32 deletions(-) create mode 100644 packages/pandas-gbq/tests/unit/conftest.py create mode 100644 packages/pandas-gbq/tests/unit/test_context.py diff --git a/packages/pandas-gbq/docs/source/api.rst b/packages/pandas-gbq/docs/source/api.rst index f5bcf9576bce..a189bae577ba 100644 --- a/packages/pandas-gbq/docs/source/api.rst +++ b/packages/pandas-gbq/docs/source/api.rst @@ -14,6 +14,10 @@ API Reference read_gbq to_gbq + context + Context .. autofunction:: read_gbq .. autofunction:: to_gbq +.. autodata:: context +.. autoclass:: Context diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 071ad529b05b..073a9f7b4c94 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,6 +1,14 @@ Changelog ========= +.. _changelog-0.7.0: + +0.7.0 / [unreleased] +-------------------- + +- Add :class:`pandas_gbq.Context` to cache credentials in-memory, across + calls to ``read_gbq`` and ``to_gbq``. (:issue:`198`, :issue:`208`) + .. _changelog-0.6.1: 0.6.1 / 2018-09-11 diff --git a/packages/pandas-gbq/pandas_gbq/__init__.py b/packages/pandas-gbq/pandas_gbq/__init__.py index febda7c65477..401d114a4761 100644 --- a/packages/pandas-gbq/pandas_gbq/__init__.py +++ b/packages/pandas-gbq/pandas_gbq/__init__.py @@ -1,4 +1,4 @@ -from .gbq import to_gbq, read_gbq # noqa +from .gbq import to_gbq, read_gbq, Context, context # noqa from ._version import get_versions diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index c45384e45858..6add4cdcd8d9 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -162,6 +162,72 @@ class TableCreationError(ValueError): pass +class Context(object): + """Storage for objects to be used throughout a session. + + A Context object is initialized when the ``pandas_gbq`` module is + imported, and can be found at :attr:`pandas_gbq.context`. + """ + + def __init__(self): + self._credentials = None + self._project = None + + @property + def credentials(self): + """google.auth.credentials.Credentials: Credentials to use for Google + APIs. + + Note: + These credentials are automatically cached in memory by calls to + :func:`pandas_gbq.read_gbq` and :func:`pandas_gbq.to_gbq`. To + manually set the credentials, construct an + :class:`google.auth.credentials.Credentials` object and set it as + the context credentials as demonstrated in the example below. See + `auth docs`_ for more information on obtaining credentials. + + Example: + Manually setting the context credentials: + >>> import pandas_gbq + >>> from google.oauth2 import service_account + >>> credentials = (service_account + ... .Credentials.from_service_account_file( + ... '/path/to/key.json')) + >>> pandas_gbq.context.credentials = credentials + .. _auth docs: http://google-auth.readthedocs.io + /en/latest/user-guide.html#obtaining-credentials + """ + return self._credentials + + @credentials.setter + def credentials(self, value): + self._credentials = value + + @property + def project(self): + """str: Default project to use for calls to Google APIs. + + Example: + Manually setting the context project: + >>> import pandas_gbq + >>> pandas_gbq.context.project = 'my-project' + """ + return self._project + + @project.setter + def project(self, value): + self._project = value + + +# Create an empty context, used to cache credentials. +context = Context() +"""A :class:`pandas_gbq.Context` object used to cache credentials. + +Credentials automatically are cached in-memory by :func:`pandas_gbq.read_gbq` +and :func:`pandas_gbq.to_gbq`. +""" + + class GbqConnector(object): def __init__( self, @@ -173,6 +239,7 @@ def __init__( location=None, try_credentials=None, ): + global context from google.api_core.exceptions import GoogleAPIError from google.api_core.exceptions import ClientError from pandas_gbq import auth @@ -185,13 +252,20 @@ def __init__( self.auth_local_webserver = auth_local_webserver self.dialect = dialect self.credentials_path = _get_credentials_file() - self.credentials, default_project = auth.get_credentials( - private_key=private_key, - project_id=project_id, - reauth=reauth, - auth_local_webserver=auth_local_webserver, - try_credentials=try_credentials, - ) + + # Load credentials from cache. + self.credentials = context.credentials + default_project = context.project + + # Credentials were explicitly asked for, so don't use the cache. + if private_key or reauth or not self.credentials: + self.credentials, default_project = auth.get_credentials( + private_key=private_key, + project_id=project_id, + reauth=reauth, + auth_local_webserver=auth_local_webserver, + try_credentials=try_credentials, + ) if self.project_id is None: self.project_id = default_project @@ -201,6 +275,12 @@ def __init__( "Could not determine project ID and one was not supplied." ) + # Cache the credentials if they haven't been set yet. + if context.credentials is None: + context.credentials = self.credentials + if context.project is None: + context.project = self.project_id + self.client = self.get_client() # BQ Queries costs $5 per TB. First 1 TB per month is free diff --git a/packages/pandas-gbq/tests/unit/conftest.py b/packages/pandas-gbq/tests/unit/conftest.py new file mode 100644 index 000000000000..ece8421d933b --- /dev/null +++ b/packages/pandas-gbq/tests/unit/conftest.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- + +try: + from unittest import mock +except ImportError: # pragma: NO COVER + import mock + +import pytest + + +@pytest.fixture(autouse=True, scope="function") +def reset_context(): + import pandas_gbq + + pandas_gbq.context.credentials = None + pandas_gbq.context.project = None + + +@pytest.fixture(autouse=True) +def mock_bigquery_client(monkeypatch): + from pandas_gbq import gbq + from google.api_core.exceptions import NotFound + import google.cloud.bigquery + import google.cloud.bigquery.table + + mock_client = mock.create_autospec(google.cloud.bigquery.Client) + mock_schema = [google.cloud.bigquery.SchemaField("_f0", "INTEGER")] + # Mock out SELECT 1 query results. + mock_query = mock.create_autospec(google.cloud.bigquery.QueryJob) + mock_query.job_id = "some-random-id" + mock_query.state = "DONE" + mock_rows = mock.create_autospec(google.cloud.bigquery.table.RowIterator) + mock_rows.total_rows = 1 + mock_rows.schema = mock_schema + mock_rows.__iter__.return_value = [(1,)] + mock_query.result.return_value = mock_rows + mock_client.query.return_value = mock_query + # Mock table creation. + mock_client.get_table.side_effect = NotFound("nope") + monkeypatch.setattr(gbq.GbqConnector, "get_client", lambda _: mock_client) + return mock_client diff --git a/packages/pandas-gbq/tests/unit/test_context.py b/packages/pandas-gbq/tests/unit/test_context.py new file mode 100644 index 000000000000..352ece7ecd09 --- /dev/null +++ b/packages/pandas-gbq/tests/unit/test_context.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- + +try: + from unittest import mock +except ImportError: # pragma: NO COVER + import mock + +import pytest + + +@pytest.fixture(autouse=True) +def mock_get_credentials(monkeypatch): + from pandas_gbq import auth + import google.auth.credentials + + mock_credentials = mock.MagicMock(google.auth.credentials.Credentials) + mock_get_credentials = mock.Mock() + mock_get_credentials.return_value = (mock_credentials, "my-project") + + monkeypatch.setattr(auth, "get_credentials", mock_get_credentials) + return mock_get_credentials + + +def test_read_gbq_should_save_credentials(mock_get_credentials): + import pandas_gbq + + assert pandas_gbq.context.credentials is None + assert pandas_gbq.context.project is None + + pandas_gbq.read_gbq("SELECT 1", dialect="standard") + + assert mock_get_credentials.call_count == 1 + mock_get_credentials.reset_mock() + assert pandas_gbq.context.credentials is not None + assert pandas_gbq.context.project is not None + + pandas_gbq.read_gbq("SELECT 1", dialect="standard") + mock_get_credentials.assert_not_called() diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index 4a42e057f78b..1f3ec9a4fd43 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -25,30 +25,6 @@ def min_bq_version(): return pkg_resources.parse_version("0.32.0") -@pytest.fixture(autouse=True) -def mock_bigquery_client(monkeypatch): - from google.api_core.exceptions import NotFound - import google.cloud.bigquery - import google.cloud.bigquery.table - - mock_client = mock.create_autospec(google.cloud.bigquery.Client) - mock_schema = [google.cloud.bigquery.SchemaField("_f0", "INTEGER")] - # Mock out SELECT 1 query results. - mock_query = mock.create_autospec(google.cloud.bigquery.QueryJob) - mock_query.job_id = "some-random-id" - mock_query.state = "DONE" - mock_rows = mock.create_autospec(google.cloud.bigquery.table.RowIterator) - mock_rows.total_rows = 1 - mock_rows.schema = mock_schema - mock_rows.__iter__.return_value = [(1,)] - mock_query.result.return_value = mock_rows - mock_client.query.return_value = mock_query - # Mock table creation. - mock_client.get_table.side_effect = NotFound("nope") - monkeypatch.setattr(gbq.GbqConnector, "get_client", lambda _: mock_client) - return mock_client - - def mock_none_credentials(*args, **kwargs): return None, None From 7edf817c9a5897523c614de6ae6a2a58a443757c Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 11 Sep 2018 10:06:21 -0700 Subject: [PATCH 149/519] [BUG] Don't load credentials from disk if reauth is True (#212) * [BUG] Don't load credentials from disk if reauth is True If the credentials on disk are invalid, then `get_user_credentials()` may fail before it can fetch fresh credentials. Noticed first and fixed in pydata-google-auth package in https://github.com/pydata/pydata-google-auth/commit/7ce7f3f822fbaea7cc73878f52c61f8b38109f36 Note about pydata-google-auth: eventually I'd like to move pandas-gbq to use that package for auth, but since I'm not sure when I'll have that package ready, I'm patching pandas-gbq, too. * syntax error --- packages/pandas-gbq/pandas_gbq/auth.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/auth.py b/packages/pandas-gbq/pandas_gbq/auth.py index 2bc2efeac971..86ad929a54c7 100644 --- a/packages/pandas-gbq/pandas_gbq/auth.py +++ b/packages/pandas-gbq/pandas_gbq/auth.py @@ -159,11 +159,13 @@ def get_user_account_credentials( if os.path.isfile("bigquery_credentials.dat"): os.rename("bigquery_credentials.dat", credentials_path) - credentials = load_user_account_credentials( - try_credentials, - project_id=project_id, - credentials_path=credentials_path, - ) + credentials = None + if not reauth: + credentials = load_user_account_credentials( + try_credentials, + project_id=project_id, + credentials_path=credentials_path, + ) client_config = { "installed": { @@ -178,7 +180,7 @@ def get_user_account_credentials( } } - if credentials is None or reauth: + if credentials is None: app_flow = InstalledAppFlow.from_client_config( client_config, scopes=SCOPES ) From 33100c0fea3651e711a855ae23e85a12889b70b2 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Thu, 13 Sep 2018 11:35:51 -0400 Subject: [PATCH 150/519] Even quieter logging for short queries (#214) * even quieter logging for short queries * changelog * more faster --- packages/pandas-gbq/docs/source/changelog.rst | 3 +++ packages/pandas-gbq/pandas_gbq/gbq.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 073a9f7b4c94..193978850faf 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -8,6 +8,9 @@ Changelog - Add :class:`pandas_gbq.Context` to cache credentials in-memory, across calls to ``read_gbq`` and ``to_gbq``. (:issue:`198`, :issue:`208`) +- Fast queries now do not log above ``DEBUG`` level. (:issue:`204`). + With BigQuery's release of `clustering `__ + querying smaller samples of data is now faster and cheaper. .. _changelog-0.6.1: diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 6add4cdcd8d9..450aa250949f 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -293,7 +293,7 @@ def _start_timer(self): def get_elapsed_seconds(self): return round(time.time() - self.start, 2) - def log_elapsed_seconds(self, prefix="Elapsed", postfix="s.", overlong=7): + def log_elapsed_seconds(self, prefix="Elapsed", postfix="s.", overlong=6): sec = self.get_elapsed_seconds() if sec > overlong: logger.info("{} {} {}".format(prefix, sec, postfix)) @@ -357,7 +357,7 @@ def run_query(self, query, **kwargs): job_config=bigquery.QueryJobConfig.from_api_repr(job_config), location=self.location, ) - logger.info("Query running...") + logger.debug("Query running...") except (RefreshError, ValueError): if self.private_key: raise AccessDenied( From 1786d2a9e4cc75951259c42584ffdf50de092243 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 14 Sep 2018 15:09:12 -0700 Subject: [PATCH 151/519] [TST] Avoid listing datasets and tables in system tests (#216) * [TST] Avoid listing datasets and tables in system tests Most tests only use a single dataset, so it doesn't make sense to list datasets in the clean-up method. Also, listing datasets is eventually-consistent, so avoiding listing when it isn't necessary makes the tests less flaky. Removes unused _Dataset.tables() and _Dataset.datasets() methods. * Remove unused randint import. * Update changelog. --- packages/pandas-gbq/docs/source/changelog.rst | 11 +- packages/pandas-gbq/pandas_gbq/gbq.py | 79 --- packages/pandas-gbq/tests/system/test_gbq.py | 646 ++++++++---------- 3 files changed, 307 insertions(+), 429 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 193978850faf..28cecbcae853 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -8,9 +8,18 @@ Changelog - Add :class:`pandas_gbq.Context` to cache credentials in-memory, across calls to ``read_gbq`` and ``to_gbq``. (:issue:`198`, :issue:`208`) -- Fast queries now do not log above ``DEBUG`` level. (:issue:`204`). +- Fast queries now do not log above ``DEBUG`` level. (:issue:`204`) With BigQuery's release of `clustering `__ querying smaller samples of data is now faster and cheaper. +- Don't load credentials from disk if reauth is ``True``. (:issue:`212`) + This fixes a bug where pandas-gbq could not refresh credentials if the + cached credentials were invalid, revoked, or expired, even when + ``reauth=True``. + +Internal changes +~~~~~~~~~~~~~~~~ + +- Avoid listing datasets and tables in system tests. (:issue:`215`) .. _changelog-0.6.1: diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 450aa250949f..32ba002c4aad 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -1088,32 +1088,6 @@ def exists(self, dataset_id): except self.http_error as ex: self.process_http_error(ex) - def datasets(self): - """ Return a list of datasets in Google BigQuery - - Parameters - ---------- - None - - Returns - ------- - list - List of datasets under the specific project - """ - - dataset_list = [] - - try: - dataset_response = self.client.list_datasets() - - for row in dataset_response: - dataset_list.append(row.dataset_id) - - except self.http_error as ex: - self.process_http_error(ex) - - return dataset_list - def create(self, dataset_id): """ Create a dataset in Google BigQuery @@ -1135,56 +1109,3 @@ def create(self, dataset_id): self.client.create_dataset(dataset) except self.http_error as ex: self.process_http_error(ex) - - def delete(self, dataset_id): - """ Delete a dataset in Google BigQuery - - Parameters - ---------- - dataset : str - Name of dataset to be deleted - """ - from google.api_core.exceptions import NotFound - - if not self.exists(dataset_id): - raise NotFoundException( - "Dataset {0} does not exist".format(dataset_id) - ) - - try: - self.client.delete_dataset(self.client.dataset(dataset_id)) - - except NotFound: - # Ignore 404 error which may occur if dataset already deleted - pass - except self.http_error as ex: - self.process_http_error(ex) - - def tables(self, dataset_id): - """ List tables in the specific dataset in Google BigQuery - - Parameters - ---------- - dataset : str - Name of dataset to list tables for - - Returns - ------- - list - List of tables under the specific dataset - """ - - table_list = [] - - try: - table_response = self.client.list_tables( - self.client.dataset(dataset_id) - ) - - for row in table_response: - table_list.append(row.table_id) - - except self.http_error as ex: - self.process_http_error(ex) - - return table_list diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index 261339f375ee..ba85b4c22ea3 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -2,7 +2,7 @@ import sys from datetime import datetime -from random import randint +import uuid import numpy as np import pandas.util.testing as tm @@ -16,10 +16,6 @@ TABLE_ID = "new_test" -def _get_dataset_prefix_random(): - return "".join(["pandas_gbq_", str(randint(1, 100000))]) - - def test_imports(): try: import pkg_resources # noqa @@ -56,20 +52,31 @@ def bigquery_client(project_id, private_key_path): ) -@pytest.fixture(scope="module") -def tokyo_dataset(bigquery_client): - from google.cloud import bigquery +@pytest.fixture() +def random_dataset_id(bigquery_client): + import google.api_core.exceptions - dataset_id = "tokyo_{}".format(_get_dataset_prefix_random()) + dataset_id = "".join(["pandas_gbq_", str(uuid.uuid4()).replace("-", "_")]) dataset_ref = bigquery_client.dataset(dataset_id) + yield dataset_id + try: + bigquery_client.delete_dataset(dataset_ref, delete_contents=True) + except google.api_core.exceptions.NotFound: + pass # Not all tests actually create a dataset + + +@pytest.fixture() +def tokyo_dataset(bigquery_client, random_dataset_id): + from google.cloud import bigquery + + dataset_ref = bigquery_client.dataset(random_dataset_id) dataset = bigquery.Dataset(dataset_ref) dataset.location = "asia-northeast1" bigquery_client.create_dataset(dataset) - yield dataset_id - bigquery_client.delete_dataset(dataset_ref, delete_contents=True) + return random_dataset_id -@pytest.fixture(scope="module") +@pytest.fixture() def tokyo_table(bigquery_client, tokyo_dataset): table_id = "tokyo_table" # Create a random table using DDL. @@ -88,11 +95,14 @@ def tokyo_table(bigquery_client, tokyo_dataset): return table_id -def clean_gbq_environment(dataset_prefix, bigquery_client): - for dataset in bigquery_client.list_datasets(): - if not dataset.dataset_id.startswith(dataset_prefix): - continue - bigquery_client.delete_dataset(dataset.reference, delete_contents=True) +@pytest.fixture() +def gbq_dataset(project, credentials): + return gbq._Dataset(project, private_key=credentials) + + +@pytest.fixture() +def gbq_table(project, credentials, random_dataset_id): + return gbq._Table(project, random_dataset_id, private_key=credentials) def make_mixed_dataframe_v2(test_size): @@ -838,32 +848,16 @@ def test_tokyo(self, tokyo_dataset, tokyo_table, private_key_path): class TestToGBQIntegration(object): - # Changes to BigQuery table schema may take up to 2 minutes as of May 2015 - # As a workaround to this issue, each test should use a unique table name. - # Make sure to modify the for loop range in the autouse fixture when a new - # test is added See `Issue 191 - # `__ - @pytest.fixture(autouse=True, scope="function") - def setup(self, project, credentials, bigquery_client): + def setup(self, project, credentials, random_dataset_id): # - PER-TEST FIXTURES - # put here any instruction you want to be run *BEFORE* *EVERY* test is # executed. - - self.dataset_prefix = _get_dataset_prefix_random() - clean_gbq_environment(self.dataset_prefix, bigquery_client) - self.dataset = gbq._Dataset(project, private_key=credentials) self.table = gbq._Table( - project, self.dataset_prefix + "1", private_key=credentials - ) - self.sut = gbq.GbqConnector(project, private_key=credentials) - self.destination_table = "{0}{1}.{2}".format( - self.dataset_prefix, "1", TABLE_ID + project, random_dataset_id, private_key=credentials ) - self.dataset.create(self.dataset_prefix + "1") + self.destination_table = "{}.{}".format(random_dataset_id, TABLE_ID) self.credentials = credentials - yield - clean_gbq_environment(self.dataset_prefix, bigquery_client) def test_upload_data(self, project_id): test_id = "1" @@ -1167,144 +1161,6 @@ def test_upload_mixed_float_and_int(self, project_id): assert len(result_df) == test_size - # TODO: move generate schema test to unit tests. - def test_generate_schema(self): - df = tm.makeMixedDataFrame() - schema = gbq._generate_bq_schema(df) - - test_schema = { - "fields": [ - {"name": "A", "type": "FLOAT"}, - {"name": "B", "type": "FLOAT"}, - {"name": "C", "type": "STRING"}, - {"name": "D", "type": "TIMESTAMP"}, - ] - } - - assert schema == test_schema - - def test_create_table(self): - test_id = "6" - schema = gbq._generate_bq_schema(tm.makeMixedDataFrame()) - self.table.create(TABLE_ID + test_id, schema) - assert self.table.exists(TABLE_ID + test_id) - - def test_create_table_already_exists(self): - test_id = "6" - schema = gbq._generate_bq_schema(tm.makeMixedDataFrame()) - self.table.create(TABLE_ID + test_id, schema) - with pytest.raises(gbq.TableCreationError): - self.table.create(TABLE_ID + test_id, schema) - - def test_table_does_not_exist(self): - test_id = "7" - assert not self.table.exists(TABLE_ID + test_id) - - def test_delete_table(self): - test_id = "8" - test_schema = { - "fields": [ - {"name": "A", "type": "FLOAT"}, - {"name": "B", "type": "FLOAT"}, - {"name": "C", "type": "STRING"}, - {"name": "D", "type": "TIMESTAMP"}, - ] - } - self.table.create(TABLE_ID + test_id, test_schema) - self.table.delete(TABLE_ID + test_id) - assert not self.table.exists(TABLE_ID + test_id) - - def test_delete_table_not_found(self): - with pytest.raises(gbq.NotFoundException): - self.table.delete(TABLE_ID + "not_found") - - def test_list_table(self): - test_id = "9" - test_schema = { - "fields": [ - {"name": "A", "type": "FLOAT"}, - {"name": "B", "type": "FLOAT"}, - {"name": "C", "type": "STRING"}, - {"name": "D", "type": "TIMESTAMP"}, - ] - } - self.table.create(TABLE_ID + test_id, test_schema) - assert TABLE_ID + test_id in self.dataset.tables( - self.dataset_prefix + "1" - ) - - def test_verify_schema_allows_flexible_column_order(self): - test_id = "10" - test_schema_1 = { - "fields": [ - {"name": "A", "type": "FLOAT"}, - {"name": "B", "type": "FLOAT"}, - {"name": "C", "type": "STRING"}, - {"name": "D", "type": "TIMESTAMP"}, - ] - } - test_schema_2 = { - "fields": [ - {"name": "A", "type": "FLOAT"}, - {"name": "C", "type": "STRING"}, - {"name": "B", "type": "FLOAT"}, - {"name": "D", "type": "TIMESTAMP"}, - ] - } - - self.table.create(TABLE_ID + test_id, test_schema_1) - assert self.sut.verify_schema( - self.dataset_prefix + "1", TABLE_ID + test_id, test_schema_2 - ) - - def test_verify_schema_fails_different_data_type(self): - test_id = "11" - test_schema_1 = { - "fields": [ - {"name": "A", "type": "FLOAT"}, - {"name": "B", "type": "FLOAT"}, - {"name": "C", "type": "STRING"}, - {"name": "D", "type": "TIMESTAMP"}, - ] - } - test_schema_2 = { - "fields": [ - {"name": "A", "type": "FLOAT"}, - {"name": "B", "type": "STRING"}, - {"name": "C", "type": "STRING"}, - {"name": "D", "type": "TIMESTAMP"}, - ] - } - - self.table.create(TABLE_ID + test_id, test_schema_1) - assert not self.sut.verify_schema( - self.dataset_prefix + "1", TABLE_ID + test_id, test_schema_2 - ) - - def test_verify_schema_fails_different_structure(self): - test_id = "12" - test_schema_1 = { - "fields": [ - {"name": "A", "type": "FLOAT"}, - {"name": "B", "type": "FLOAT"}, - {"name": "C", "type": "STRING"}, - {"name": "D", "type": "TIMESTAMP"}, - ] - } - test_schema_2 = { - "fields": [ - {"name": "A", "type": "FLOAT"}, - {"name": "B2", "type": "FLOAT"}, - {"name": "C", "type": "STRING"}, - {"name": "D", "type": "TIMESTAMP"}, - ] - } - - self.table.create(TABLE_ID + test_id, test_schema_1) - assert not self.sut.verify_schema( - self.dataset_prefix + "1", TABLE_ID + test_id, test_schema_2 - ) - def test_upload_data_flexible_column_order(self, project_id): test_id = "13" test_size = 10 @@ -1329,131 +1185,6 @@ def test_upload_data_flexible_column_order(self, project_id): private_key=self.credentials, ) - def test_verify_schema_ignores_field_mode(self): - test_id = "14" - test_schema_1 = { - "fields": [ - {"name": "A", "type": "FLOAT", "mode": "NULLABLE"}, - {"name": "B", "type": "FLOAT", "mode": "NULLABLE"}, - {"name": "C", "type": "STRING", "mode": "NULLABLE"}, - {"name": "D", "type": "TIMESTAMP", "mode": "REQUIRED"}, - ] - } - test_schema_2 = { - "fields": [ - {"name": "A", "type": "FLOAT"}, - {"name": "B", "type": "FLOAT"}, - {"name": "C", "type": "STRING"}, - {"name": "D", "type": "TIMESTAMP"}, - ] - } - - self.table.create(TABLE_ID + test_id, test_schema_1) - assert self.sut.verify_schema( - self.dataset_prefix + "1", TABLE_ID + test_id, test_schema_2 - ) - - def test_retrieve_schema(self): - # Issue #24 schema function returns the schema in biquery - test_id = "15" - test_schema = { - "fields": [ - { - "name": "A", - "type": "FLOAT", - "mode": "NULLABLE", - "description": None, - }, - { - "name": "B", - "type": "FLOAT", - "mode": "NULLABLE", - "description": None, - }, - { - "name": "C", - "type": "STRING", - "mode": "NULLABLE", - "description": None, - }, - { - "name": "D", - "type": "TIMESTAMP", - "mode": "NULLABLE", - "description": None, - }, - ] - } - - self.table.create(TABLE_ID + test_id, test_schema) - actual = self.sut._clean_schema_fields( - self.sut.schema(self.dataset_prefix + "1", TABLE_ID + test_id) - ) - expected = [ - {"name": "A", "type": "FLOAT"}, - {"name": "B", "type": "FLOAT"}, - {"name": "C", "type": "STRING"}, - {"name": "D", "type": "TIMESTAMP"}, - ] - assert expected == actual, "Expected schema used to create table" - - def test_schema_is_subset_passes_if_subset(self): - # Issue #24 schema_is_subset indicates whether the schema of the - # dataframe is a subset of the schema of the bigquery table - test_id = "16" - - table_name = TABLE_ID + test_id - dataset = self.dataset_prefix + "1" - - table_schema = { - "fields": [ - {"name": "A", "type": "FLOAT"}, - {"name": "B", "type": "FLOAT"}, - {"name": "C", "type": "STRING"}, - ] - } - tested_schema = { - "fields": [ - {"name": "A", "type": "FLOAT"}, - {"name": "B", "type": "FLOAT"}, - ] - } - - self.table.create(table_name, table_schema) - - assert ( - self.sut.schema_is_subset(dataset, table_name, tested_schema) - is True - ) - - def test_schema_is_subset_fails_if_not_subset(self): - # For pull request #24 - test_id = "17" - - table_name = TABLE_ID + test_id - dataset = self.dataset_prefix + "1" - - table_schema = { - "fields": [ - {"name": "A", "type": "FLOAT"}, - {"name": "B", "type": "FLOAT"}, - {"name": "C", "type": "STRING"}, - ] - } - tested_schema = { - "fields": [ - {"name": "A", "type": "FLOAT"}, - {"name": "C", "type": "FLOAT"}, - ] - } - - self.table.create(table_name, table_schema) - - assert ( - self.sut.schema_is_subset(dataset, table_name, tested_schema) - is False - ) - def test_upload_data_with_valid_user_schema(self, project_id): # Issue #46; tests test scenarios with user-provided # schemas @@ -1594,53 +1325,270 @@ def test_upload_data_tokyo( ) assert table.num_rows > 0 - def test_list_dataset(self): - dataset_id = self.dataset_prefix + "1" - assert dataset_id in self.dataset.datasets() - - def test_list_table_zero_results(self, project_id): - dataset_id = self.dataset_prefix + "2" - self.dataset.create(dataset_id) - table_list = gbq._Dataset( - project_id, private_key=self.credentials - ).tables(dataset_id) - assert len(table_list) == 0 - - def test_create_dataset(self): - dataset_id = self.dataset_prefix + "3" - self.dataset.create(dataset_id) - assert dataset_id in self.dataset.datasets() - - def test_create_dataset_already_exists(self): - dataset_id = self.dataset_prefix + "3" - self.dataset.create(dataset_id) - with pytest.raises(gbq.DatasetCreationError): - self.dataset.create(dataset_id) - - def test_delete_dataset(self): - dataset_id = self.dataset_prefix + "4" - self.dataset.create(dataset_id) - self.dataset.delete(dataset_id) - assert dataset_id not in self.dataset.datasets() - - def test_delete_dataset_not_found(self): - dataset_id = self.dataset_prefix + "not_found" - with pytest.raises(gbq.NotFoundException): - self.dataset.delete(dataset_id) - - def test_dataset_exists(self): - dataset_id = self.dataset_prefix + "5" - self.dataset.create(dataset_id) - assert self.dataset.exists(dataset_id) - - def create_table_data_dataset_does_not_exist(self, project_id): - dataset_id = self.dataset_prefix + "6" - table_id = TABLE_ID + "1" - table_with_new_dataset = gbq._Table(project_id, dataset_id) - df = make_mixed_dataframe_v2(10) - table_with_new_dataset.create(table_id, gbq._generate_bq_schema(df)) - assert self.dataset.exists(dataset_id) - assert table_with_new_dataset.exists(table_id) - - def test_dataset_does_not_exist(self): - assert not self.dataset.exists(self.dataset_prefix + "_not_found") + +# _Dataset tests + + +def test_create_dataset(bigquery_client, gbq_dataset, random_dataset_id): + gbq_dataset.create(random_dataset_id) + dataset_reference = bigquery_client.dataset(random_dataset_id) + assert bigquery_client.get_dataset(dataset_reference) is not None + + +def test_create_dataset_already_exists(gbq_dataset, random_dataset_id): + gbq_dataset.create(random_dataset_id) + with pytest.raises(gbq.DatasetCreationError): + gbq_dataset.create(random_dataset_id) + + +def test_dataset_exists(gbq_dataset, random_dataset_id): + gbq_dataset.create(random_dataset_id) + assert gbq_dataset.exists(random_dataset_id) + + +def test_dataset_does_not_exist(gbq_dataset, random_dataset_id): + assert not gbq_dataset.exists(random_dataset_id) + + +# _Table tests + + +def test_create_table(gbq_table): + schema = gbq._generate_bq_schema(tm.makeMixedDataFrame()) + gbq_table.create("test_create_table", schema) + assert gbq_table.exists("test_create_table") + + +def test_create_table_already_exists(gbq_table): + schema = gbq._generate_bq_schema(tm.makeMixedDataFrame()) + gbq_table.create("test_create_table_exists", schema) + with pytest.raises(gbq.TableCreationError): + gbq_table.create("test_create_table_exists", schema) + + +def test_table_does_not_exist(gbq_table): + assert not gbq_table.exists("test_table_does_not_exist") + + +def test_delete_table(gbq_table): + test_schema = { + "fields": [ + {"name": "A", "type": "FLOAT"}, + {"name": "B", "type": "FLOAT"}, + {"name": "C", "type": "STRING"}, + {"name": "D", "type": "TIMESTAMP"}, + ] + } + gbq_table.create("test_delete_table", test_schema) + gbq_table.delete("test_delete_table") + assert not gbq_table.exists("test_delete_table") + + +def test_delete_table_not_found(gbq_table): + with pytest.raises(gbq.NotFoundException): + gbq_table.delete("test_delete_table_not_found") + + +def test_create_table_data_dataset_does_not_exist( + project, credentials, gbq_dataset, random_dataset_id +): + table_id = "test_create_table_data_dataset_does_not_exist" + table_with_new_dataset = gbq._Table( + project, random_dataset_id, private_key=credentials + ) + df = make_mixed_dataframe_v2(10) + table_with_new_dataset.create(table_id, gbq._generate_bq_schema(df)) + assert gbq_dataset.exists(random_dataset_id) + assert table_with_new_dataset.exists(table_id) + + +def test_verify_schema_allows_flexible_column_order(gbq_table, gbq_connector): + table_id = "test_verify_schema_allows_flexible_column_order" + test_schema_1 = { + "fields": [ + {"name": "A", "type": "FLOAT"}, + {"name": "B", "type": "FLOAT"}, + {"name": "C", "type": "STRING"}, + {"name": "D", "type": "TIMESTAMP"}, + ] + } + test_schema_2 = { + "fields": [ + {"name": "A", "type": "FLOAT"}, + {"name": "C", "type": "STRING"}, + {"name": "B", "type": "FLOAT"}, + {"name": "D", "type": "TIMESTAMP"}, + ] + } + + gbq_table.create(table_id, test_schema_1) + assert gbq_connector.verify_schema( + gbq_table.dataset_id, table_id, test_schema_2 + ) + + +def test_verify_schema_fails_different_data_type(gbq_table, gbq_connector): + table_id = "test_verify_schema_fails_different_data_type" + test_schema_1 = { + "fields": [ + {"name": "A", "type": "FLOAT"}, + {"name": "B", "type": "FLOAT"}, + {"name": "C", "type": "STRING"}, + {"name": "D", "type": "TIMESTAMP"}, + ] + } + test_schema_2 = { + "fields": [ + {"name": "A", "type": "FLOAT"}, + {"name": "B", "type": "STRING"}, + {"name": "C", "type": "STRING"}, + {"name": "D", "type": "TIMESTAMP"}, + ] + } + + gbq_table.create(table_id, test_schema_1) + assert not gbq_connector.verify_schema( + gbq_table.dataset_id, table_id, test_schema_2 + ) + + +def test_verify_schema_fails_different_structure(gbq_table, gbq_connector): + table_id = "test_verify_schema_fails_different_structure" + test_schema_1 = { + "fields": [ + {"name": "A", "type": "FLOAT"}, + {"name": "B", "type": "FLOAT"}, + {"name": "C", "type": "STRING"}, + {"name": "D", "type": "TIMESTAMP"}, + ] + } + test_schema_2 = { + "fields": [ + {"name": "A", "type": "FLOAT"}, + {"name": "B2", "type": "FLOAT"}, + {"name": "C", "type": "STRING"}, + {"name": "D", "type": "TIMESTAMP"}, + ] + } + + gbq_table.create(table_id, test_schema_1) + assert not gbq_connector.verify_schema( + gbq_table.dataset_id, table_id, test_schema_2 + ) + + +def test_verify_schema_ignores_field_mode(gbq_table, gbq_connector): + table_id = "test_verify_schema_ignores_field_mode" + test_schema_1 = { + "fields": [ + {"name": "A", "type": "FLOAT", "mode": "NULLABLE"}, + {"name": "B", "type": "FLOAT", "mode": "NULLABLE"}, + {"name": "C", "type": "STRING", "mode": "NULLABLE"}, + {"name": "D", "type": "TIMESTAMP", "mode": "REQUIRED"}, + ] + } + test_schema_2 = { + "fields": [ + {"name": "A", "type": "FLOAT"}, + {"name": "B", "type": "FLOAT"}, + {"name": "C", "type": "STRING"}, + {"name": "D", "type": "TIMESTAMP"}, + ] + } + + gbq_table.create(table_id, test_schema_1) + assert gbq_connector.verify_schema( + gbq_table.dataset_id, table_id, test_schema_2 + ) + + +def test_retrieve_schema(gbq_table, gbq_connector): + # Issue #24 schema function returns the schema in biquery + table_id = "test_retrieve_schema" + test_schema = { + "fields": [ + { + "name": "A", + "type": "FLOAT", + "mode": "NULLABLE", + "description": None, + }, + { + "name": "B", + "type": "FLOAT", + "mode": "NULLABLE", + "description": None, + }, + { + "name": "C", + "type": "STRING", + "mode": "NULLABLE", + "description": None, + }, + { + "name": "D", + "type": "TIMESTAMP", + "mode": "NULLABLE", + "description": None, + }, + ] + } + + gbq_table.create(table_id, test_schema) + actual = gbq_connector._clean_schema_fields( + gbq_connector.schema(gbq_table.dataset_id, table_id) + ) + expected = [ + {"name": "A", "type": "FLOAT"}, + {"name": "B", "type": "FLOAT"}, + {"name": "C", "type": "STRING"}, + {"name": "D", "type": "TIMESTAMP"}, + ] + assert expected == actual, "Expected schema used to create table" + + +def test_schema_is_subset_passes_if_subset(gbq_table, gbq_connector): + # Issue #24 schema_is_subset indicates whether the schema of the + # dataframe is a subset of the schema of the bigquery table + table_id = "test_schema_is_subset_passes_if_subset" + table_schema = { + "fields": [ + {"name": "A", "type": "FLOAT"}, + {"name": "B", "type": "FLOAT"}, + {"name": "C", "type": "STRING"}, + ] + } + tested_schema = { + "fields": [ + {"name": "A", "type": "FLOAT"}, + {"name": "B", "type": "FLOAT"}, + ] + } + + gbq_table.create(table_id, table_schema) + assert gbq_connector.schema_is_subset( + gbq_table.dataset_id, table_id, tested_schema + ) + + +def test_schema_is_subset_fails_if_not_subset(gbq_table, gbq_connector): + # For pull request #24 + table_id = "test_schema_is_subset_fails_if_not_subset" + table_schema = { + "fields": [ + {"name": "A", "type": "FLOAT"}, + {"name": "B", "type": "FLOAT"}, + {"name": "C", "type": "STRING"}, + ] + } + tested_schema = { + "fields": [ + {"name": "A", "type": "FLOAT"}, + {"name": "C", "type": "FLOAT"}, + ] + } + + gbq_table.create(table_id, table_schema) + assert not gbq_connector.schema_is_subset( + gbq_table.dataset_id, table_id, tested_schema + ) From 0fda5fa89ac185bb46cdbb05225e856b0f2f619a Mon Sep 17 00:00:00 2001 From: melissachang <10929390+melissachang@users.noreply.github.com> Date: Wed, 19 Sep 2018 09:26:38 -0700 Subject: [PATCH 152/519] Add more documentation to to_gbq() table_schema (#217) * Add more documentation to to_gbq() table_schema * Add note about _generate_bq_schema() not preserving column order * black * pandas_gbq.gbq._generate_bq_schema() --- packages/pandas-gbq/pandas_gbq/gbq.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 32ba002c4aad..79cd1ababf44 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -1,4 +1,3 @@ - import logging import os import time @@ -13,7 +12,6 @@ logger = logging.getLogger(__name__) - BIGQUERY_INSTALLED_VERSION = None SHOW_VERBOSE_DEPRECATION = False @@ -837,9 +835,13 @@ def to_gbq( table_schema : list of dicts, optional List of BigQuery table fields to which according DataFrame columns conform to, e.g. ``[{'name': 'col1', 'type': - 'STRING'},...]``. If schema is not provided, it will be - generated according to dtypes of DataFrame columns. See - BigQuery API documentation on available names of a field. + 'STRING'},...]``. + If schema is not provided, it will be + generated according to dtypes of DataFrame columns. + If schema is provided, it must contain all DataFrame columns. + pandas_gbq.gbq._generate_bq_schema() may be used to create an initial + schema, though it doesn't preserve column order. + See BigQuery API documentation on available names of a field. .. versionadded:: 0.3.1 location : str, optional From 0716e5fa65b643c3016652aefea2ef3cd15f2464 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 21 Sep 2018 12:23:48 -0700 Subject: [PATCH 153/519] Use latest version of Nox for tests. (#222) * Use latest version of Nox for tests. Also, update lint session to use black to check. * Use plain pytest for Python 2.7 --- packages/pandas-gbq/.travis.yml | 14 +- .../pandas-gbq/ci/requirements-3.5-0.18.1.pip | 1 + packages/pandas-gbq/docs/source/conf.py | 118 ++++---- packages/pandas-gbq/{nox.py => noxfile.py} | 49 ++- packages/pandas-gbq/pandas_gbq/_version.py | 163 ++++++---- packages/pandas-gbq/versioneer.py | 283 +++++++++++------- 6 files changed, 379 insertions(+), 249 deletions(-) rename packages/pandas-gbq/{nox.py => noxfile.py} (69%) diff --git a/packages/pandas-gbq/.travis.yml b/packages/pandas-gbq/.travis.yml index a9546b88b77a..a92996cdb7a9 100644 --- a/packages/pandas-gbq/.travis.yml +++ b/packages/pandas-gbq/.travis.yml @@ -4,7 +4,7 @@ matrix: include: - os: linux python: 2.7 - env: PYTHON=2.7 PANDAS=0.19.2 COVERAGE='false' LINT='true' + env: PYTHON=2.7 PANDAS=0.19.2 COVERAGE='false' LINT='false' - os: linux python: 3.5 env: PYTHON=3.5 PANDAS=0.18.1 COVERAGE='true' LINT='false' @@ -28,7 +28,7 @@ install: - pip install --upgrade pip - REQ="ci/requirements-${PYTHON}-${PANDAS}" ; if [ -f "$REQ.pip" ]; then - pip install --upgrade nox-automation ; + pip install --upgrade nox ; else wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; bash miniconda.sh -b -p $HOME/miniconda ; @@ -49,9 +49,13 @@ install: fi script: - - if [[ $PYTHON == '2.7' ]]; then nox -s test27 ; fi - - if [[ $PYTHON == '3.5' ]]; then nox -s test35 ; fi - - if [[ $PYTHON == '3.6' ]] && [[ "$PANDAS" == "MASTER" ]]; then nox -s test36master ; fi + - if [[ $PYTHON == '2.7' ]]; then + pip install -r ci/requirements-2.7-0.19.2.pip ; + pip install -e . ; + pytest tests/unit ; + fi + - if [[ $PYTHON == '3.5' ]]; then nox -s test_earliest_deps ; fi + - if [[ $PYTHON == '3.6' ]] && [[ "$PANDAS" == "MASTER" ]]; then nox -s test_latest_deps ; fi - REQ="ci/requirements-${PYTHON}-${PANDAS}" ; if [ -f "$REQ.conda" ]; then pytest --quiet -m 'not local_auth' -v tests ; diff --git a/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip b/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip index dc651977af74..116268c7bc1a 100644 --- a/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip +++ b/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip @@ -1,3 +1,4 @@ google-auth==1.4.1 google-auth-oauthlib==0.0.1 google-cloud-bigquery==0.32.0 +pandas==0.18.1 \ No newline at end of file diff --git a/packages/pandas-gbq/docs/source/conf.py b/packages/pandas-gbq/docs/source/conf.py index 3ad2767fa468..40c3911443f2 100644 --- a/packages/pandas-gbq/docs/source/conf.py +++ b/packages/pandas-gbq/docs/source/conf.py @@ -30,48 +30,49 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinx.ext.autodoc', - 'sphinx.ext.autosummary', - 'sphinx.ext.doctest', - 'sphinx.ext.extlinks', - 'sphinx.ext.todo', - 'numpydoc', # used to parse numpy-style docstrings for autodoc - 'IPython.sphinxext.ipython_console_highlighting', - 'IPython.sphinxext.ipython_directive', - 'sphinx.ext.intersphinx', - 'sphinx.ext.coverage', - 'sphinx.ext.ifconfig', - ] +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.doctest", + "sphinx.ext.extlinks", + "sphinx.ext.todo", + "numpydoc", # used to parse numpy-style docstrings for autodoc + "IPython.sphinxext.ipython_console_highlighting", + "IPython.sphinxext.ipython_directive", + "sphinx.ext.intersphinx", + "sphinx.ext.coverage", + "sphinx.ext.ifconfig", +] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. # # source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = u'pandas-gbq' -copyright = u'2017, PyData Development Team' -author = u'PyData Development Team' +project = u"pandas-gbq" +copyright = u"2017, PyData Development Team" +author = u"PyData Development Team" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = u'0.1.0' +version = u"0.1.0" # The full version, including alpha/beta/rc tags. -release = u'0.1.0' +release = u"0.1.0" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -92,7 +93,7 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This patterns also effect to html_static_path and html_extra_path -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # The reST default role (used for this markup: `text`) to use for all # documents. @@ -114,7 +115,7 @@ # show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. # modindex_common_prefix = [] @@ -130,11 +131,12 @@ # Taken from docs.readthedocs.io: # on_rtd is whether we are on readthedocs.io -on_rtd = os.environ.get('READTHEDOCS', None) == 'True' +on_rtd = os.environ.get("READTHEDOCS", None) == "True" if not on_rtd: # only import and set the theme if we're building docs locally import sphinx_rtd_theme - html_theme = 'sphinx_rtd_theme' + + html_theme = "sphinx_rtd_theme" html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # The theme to use for HTML and HTML Help pages. See the documentation for @@ -174,7 +176,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied @@ -254,34 +256,36 @@ # html_search_scorer = 'scorer.js' # Output file base name for HTML help builder. -htmlhelp_basename = 'pandas-gbqdoc' +htmlhelp_basename = "pandas-gbqdoc" # -- Options for LaTeX output --------------------------------------------- latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', - - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'pandas-gbq.tex', u'pandas-gbq Documentation', - u'PyData Development Team', 'manual'), + ( + master_doc, + "pandas-gbq.tex", + u"pandas-gbq Documentation", + u"PyData Development Team", + "manual", + ) ] # The name of an image file (relative to this directory) to place at the top of @@ -322,8 +326,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - (master_doc, 'pandas-gbq', u'pandas-gbq Documentation', - [author], 1) + (master_doc, "pandas-gbq", u"pandas-gbq Documentation", [author], 1) ] # If true, show URL addresses after external links. @@ -337,9 +340,15 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'pandas-gbq', u'pandas-gbq Documentation', - author, 'pandas-gbq', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "pandas-gbq", + u"pandas-gbq Documentation", + author, + "pandas-gbq", + "One line description of project.", + "Miscellaneous", + ) ] # Documents to append as an appendix to all manuals. @@ -361,11 +370,12 @@ # Configuration for intersphinx: intersphinx_mapping = { - 'https://docs.python.org/': None, - 'https://pandas.pydata.org/pandas-docs/stable/': None, - 'https://google-auth.readthedocs.io/en/latest/': None, + "https://docs.python.org/": None, + "https://pandas.pydata.org/pandas-docs/stable/": None, + "https://google-auth.readthedocs.io/en/latest/": None, } -extlinks = {'issue': ('https://github.com/pydata/pandas-gbq/issues/%s', - 'GH#'), - 'pr': ('https://github.com/pydata/pandas-gbq/pull/%s', 'GH#')} +extlinks = { + "issue": ("https://github.com/pydata/pandas-gbq/issues/%s", "GH#"), + "pr": ("https://github.com/pydata/pandas-gbq/pull/%s", "GH#"), +} diff --git a/packages/pandas-gbq/nox.py b/packages/pandas-gbq/noxfile.py similarity index 69% rename from packages/pandas-gbq/nox.py rename to packages/pandas-gbq/noxfile.py index 9731e438d25b..7a76559e26b2 100644 --- a/packages/pandas-gbq/nox.py +++ b/packages/pandas-gbq/noxfile.py @@ -14,8 +14,11 @@ ) +latest_python = "3.6" + + @nox.session -def default(session): +def test(session): session.install("mock", "pytest", "pytest-cov") session.install("-e", ".") @@ -53,54 +56,38 @@ def unit(session): @nox.session -def test27(session): - session.interpreter = "python2.7" - session.install( - "-r", os.path.join(".", "ci", "requirements-2.7-0.19.2.pip") - ) - default(session) - - -@nox.session -def test35(session): - session.interpreter = "python3.5" +def test_earliest_deps(session, python="3.5"): session.install( "-r", os.path.join(".", "ci", "requirements-3.5-0.18.1.pip") ) - default(session) + test(session) @nox.session -def test36(session): - session.interpreter = "python3.6" - session.install( - "-r", os.path.join(".", "ci", "requirements-3.6-0.20.1.conda") - ) - default(session) - - -@nox.session -def test36master(session): - session.interpreter = "python3.6" +def test_latest_deps(session, python=latest_python): session.install( "--pre", "--upgrade", "--timeout=60", "-f", PANDAS_PRE_WHEELS, "pandas" ) session.install( "-r", os.path.join(".", "ci", "requirements-3.6-MASTER.pip") ) - default(session) + test(session) @nox.session -def lint(session): - session.install("flake8") - session.run("flake8", "pandas_gbq", "tests", "-v") +def lint(session, python=latest_python): + session.install("black") + session.run( + "black", + "--check", + "--exclude", + "(\.git|\.hg|\.mypy_cache|\.tox|\.nox|\.venv|_build|buck-out|build|dist)", + ".", + ) @nox.session -def cover(session): - session.interpreter = "python3.5" - +def cover(session, python=latest_python): session.install("coverage", "pytest-cov") session.run("coverage", "report", "--show-missing", "--fail-under=40") session.run("coverage", "erase") diff --git a/packages/pandas-gbq/pandas_gbq/_version.py b/packages/pandas-gbq/pandas_gbq/_version.py index e4e698ea4973..0183df38f84e 100644 --- a/packages/pandas-gbq/pandas_gbq/_version.py +++ b/packages/pandas-gbq/pandas_gbq/_version.py @@ -58,17 +58,20 @@ class NotThisMethod(Exception): def register_vcs_handler(vcs, method): # decorator """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): """Store f in HANDLERS[vcs][method].""" if vcs not in HANDLERS: HANDLERS[vcs] = {} HANDLERS[vcs][method] = f return f + return decorate -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): +def run_command( + commands, args, cwd=None, verbose=False, hide_stderr=False, env=None +): """Call the given command(s).""" assert isinstance(commands, list) p = None @@ -76,10 +79,13 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, try: dispcmd = str([c] + args) # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) + p = subprocess.Popen( + [c] + args, + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr else None), + ) break except EnvironmentError: e = sys.exc_info()[1] @@ -116,16 +122,22 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): for i in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} + return { + "version": dirname[len(parentdir_prefix) :], + "full-revisionid": None, + "dirty": False, + "error": None, + "date": None, + } else: rootdirs.append(root) root = os.path.dirname(root) # up a level if verbose: - print("Tried directories %s but none started with prefix %s" % - (str(rootdirs), parentdir_prefix)) + print( + "Tried directories %s but none started with prefix %s" + % (str(rootdirs), parentdir_prefix) + ) raise NotThisMethod("rootdir doesn't start with parentdir_prefix") @@ -181,7 +193,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -190,7 +202,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) + tags = set([r for r in refs if re.search(r"\d", r)]) if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: @@ -198,19 +210,26 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] + r = ref[len(tag_prefix) :] if verbose: print("picking %s" % r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} + return { + "version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": None, + "date": date, + } # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} + return { + "version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": "no suitable tags", + "date": None, + } @register_vcs_handler("git", "pieces_from_vcs") @@ -225,8 +244,9 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) + out, rc = run_command( + GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True + ) if rc != 0: if verbose: print("Directory %s not under git control" % root) @@ -234,10 +254,19 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%s*" % tag_prefix], - cwd=root) + describe_out, rc = run_command( + GITS, + [ + "describe", + "--tags", + "--dirty", + "--always", + "--long", + "--match", + "%s*" % tag_prefix, + ], + cwd=root, + ) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") @@ -260,17 +289,18 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): dirty = git_describe.endswith("-dirty") pieces["dirty"] = dirty if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] + git_describe = git_describe[: git_describe.rindex("-dirty")] # now we have TAG-NUM-gHEX or HEX if "-" in git_describe: # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) if not mo: # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%s'" - % describe_out) + pieces["error"] = ( + "unable to parse git-describe output: '%s'" % describe_out + ) return pieces # tag @@ -279,10 +309,12 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if verbose: fmt = "tag '%s' doesn't start with prefix '%s'" print(fmt % (full_tag, tag_prefix)) - pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" - % (full_tag, tag_prefix)) + pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( + full_tag, + tag_prefix, + ) return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] + pieces["closest-tag"] = full_tag[len(tag_prefix) :] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) @@ -293,13 +325,15 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): else: # HEX: no tags pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) + count_out, rc = run_command( + GITS, ["rev-list", "HEAD", "--count"], cwd=root + ) pieces["distance"] = int(count_out) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], - cwd=root)[0].strip() + date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ + 0 + ].strip() pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces @@ -330,8 +364,7 @@ def render_pep440(pieces): rendered += ".dirty" else: # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) + rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered @@ -445,11 +478,13 @@ def render_git_describe_long(pieces): def render(pieces, style): """Render the given version pieces into the requested style.""" if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} + return { + "version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None, + } if not style or style == "default": style = "pep440" # the default @@ -469,9 +504,13 @@ def render(pieces, style): else: raise ValueError("unknown style '%s'" % style) - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} + return { + "version": rendered, + "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], + "error": None, + "date": pieces.get("date"), + } def get_versions(): @@ -485,8 +524,9 @@ def get_versions(): verbose = cfg.verbose try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, - verbose) + return git_versions_from_keywords( + get_keywords(), cfg.tag_prefix, verbose + ) except NotThisMethod: pass @@ -495,13 +535,16 @@ def get_versions(): # versionfile_source is the relative path from the top of the source # tree (where the .git directory might live) to this file. Invert # this to find the root from __file__. - for i in cfg.versionfile_source.split('/'): + for i in cfg.versionfile_source.split("/"): root = os.path.dirname(root) except NameError: - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None} + return { + "version": "0+unknown", + "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree", + "date": None, + } try: pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) @@ -515,6 +558,10 @@ def get_versions(): except NotThisMethod: pass - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", "date": None} + return { + "version": "0+unknown", + "full-revisionid": None, + "dirty": None, + "error": "unable to compute version", + "date": None, + } diff --git a/packages/pandas-gbq/versioneer.py b/packages/pandas-gbq/versioneer.py index dffd66b69a6d..0c6daedfdb44 100644 --- a/packages/pandas-gbq/versioneer.py +++ b/packages/pandas-gbq/versioneer.py @@ -310,11 +310,13 @@ def get_root(): setup_py = os.path.join(root, "setup.py") versioneer_py = os.path.join(root, "versioneer.py") if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - err = ("Versioneer was unable to run the project root directory. " - "Versioneer requires setup.py to be executed from " - "its immediate directory (like 'python setup.py COMMAND'), " - "or in a way that lets it use sys.argv[0] to find the root " - "(like 'python path/to/setup.py COMMAND').") + err = ( + "Versioneer was unable to run the project root directory. " + "Versioneer requires setup.py to be executed from " + "its immediate directory (like 'python setup.py COMMAND'), " + "or in a way that lets it use sys.argv[0] to find the root " + "(like 'python path/to/setup.py COMMAND')." + ) raise VersioneerBadRootError(err) try: # Certain runtime workflows (setup.py install/develop in a setuptools @@ -327,8 +329,10 @@ def get_root(): me_dir = os.path.normcase(os.path.splitext(me)[0]) vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) if me_dir != vsr_dir: - print("Warning: build in %s is using versioneer.py from %s" - % (os.path.dirname(me), versioneer_py)) + print( + "Warning: build in %s is using versioneer.py from %s" + % (os.path.dirname(me), versioneer_py) + ) except NameError: pass return root @@ -350,6 +354,7 @@ def get(parser, name): if parser.has_option("versioneer", name): return parser.get("versioneer", name) return None + cfg = VersioneerConfig() cfg.VCS = VCS cfg.style = get(parser, "style") or "" @@ -374,17 +379,20 @@ class NotThisMethod(Exception): def register_vcs_handler(vcs, method): # decorator """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): """Store f in HANDLERS[vcs][method].""" if vcs not in HANDLERS: HANDLERS[vcs] = {} HANDLERS[vcs][method] = f return f + return decorate -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): +def run_command( + commands, args, cwd=None, verbose=False, hide_stderr=False, env=None +): """Call the given command(s).""" assert isinstance(commands, list) p = None @@ -392,10 +400,13 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, try: dispcmd = str([c] + args) # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) + p = subprocess.Popen( + [c] + args, + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr else None), + ) break except EnvironmentError: e = sys.exc_info()[1] @@ -420,7 +431,9 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, return stdout, p.returncode -LONG_VERSION_PY['git'] = ''' +LONG_VERSION_PY[ + "git" +] = ''' # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (built by setup.py sdist) and build @@ -995,7 +1008,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -1004,7 +1017,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) + tags = set([r for r in refs if re.search(r"\d", r)]) if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: @@ -1012,19 +1025,26 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] + r = ref[len(tag_prefix) :] if verbose: print("picking %s" % r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} + return { + "version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": None, + "date": date, + } # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} + return { + "version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": "no suitable tags", + "date": None, + } @register_vcs_handler("git", "pieces_from_vcs") @@ -1039,8 +1059,9 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) + out, rc = run_command( + GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True + ) if rc != 0: if verbose: print("Directory %s not under git control" % root) @@ -1048,10 +1069,19 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%s*" % tag_prefix], - cwd=root) + describe_out, rc = run_command( + GITS, + [ + "describe", + "--tags", + "--dirty", + "--always", + "--long", + "--match", + "%s*" % tag_prefix, + ], + cwd=root, + ) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") @@ -1074,17 +1104,18 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): dirty = git_describe.endswith("-dirty") pieces["dirty"] = dirty if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] + git_describe = git_describe[: git_describe.rindex("-dirty")] # now we have TAG-NUM-gHEX or HEX if "-" in git_describe: # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) if not mo: # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%s'" - % describe_out) + pieces["error"] = ( + "unable to parse git-describe output: '%s'" % describe_out + ) return pieces # tag @@ -1093,10 +1124,12 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if verbose: fmt = "tag '%s' doesn't start with prefix '%s'" print(fmt % (full_tag, tag_prefix)) - pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" - % (full_tag, tag_prefix)) + pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( + full_tag, + tag_prefix, + ) return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] + pieces["closest-tag"] = full_tag[len(tag_prefix) :] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) @@ -1107,13 +1140,15 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): else: # HEX: no tags pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) + count_out, rc = run_command( + GITS, ["rev-list", "HEAD", "--count"], cwd=root + ) pieces["distance"] = int(count_out) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], - cwd=root)[0].strip() + date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ + 0 + ].strip() pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces @@ -1169,16 +1204,22 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): for i in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} + return { + "version": dirname[len(parentdir_prefix) :], + "full-revisionid": None, + "dirty": False, + "error": None, + "date": None, + } else: rootdirs.append(root) root = os.path.dirname(root) # up a level if verbose: - print("Tried directories %s but none started with prefix %s" % - (str(rootdirs), parentdir_prefix)) + print( + "Tried directories %s but none started with prefix %s" + % (str(rootdirs), parentdir_prefix) + ) raise NotThisMethod("rootdir doesn't start with parentdir_prefix") @@ -1207,11 +1248,17 @@ def versions_from_file(filename): contents = f.read() except EnvironmentError: raise NotThisMethod("unable to read _version.py") - mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", - contents, re.M | re.S) + mo = re.search( + r"version_json = '''\n(.*)''' # END VERSION_JSON", + contents, + re.M | re.S, + ) if not mo: - mo = re.search(r"version_json = '''\r\n(.*)''' # END VERSION_JSON", - contents, re.M | re.S) + mo = re.search( + r"version_json = '''\r\n(.*)''' # END VERSION_JSON", + contents, + re.M | re.S, + ) if not mo: raise NotThisMethod("no version_json in _version.py") return json.loads(mo.group(1)) @@ -1220,8 +1267,9 @@ def versions_from_file(filename): def write_to_version_file(filename, versions): """Write the given version number to the given _version.py file.""" os.unlink(filename) - contents = json.dumps(versions, sort_keys=True, - indent=1, separators=(",", ": ")) + contents = json.dumps( + versions, sort_keys=True, indent=1, separators=(",", ": ") + ) with open(filename, "w") as f: f.write(SHORT_VERSION_PY % contents) @@ -1253,8 +1301,7 @@ def render_pep440(pieces): rendered += ".dirty" else: # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) + rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered @@ -1368,11 +1415,13 @@ def render_git_describe_long(pieces): def render(pieces, style): """Render the given version pieces into the requested style.""" if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} + return { + "version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None, + } if not style or style == "default": style = "pep440" # the default @@ -1392,9 +1441,13 @@ def render(pieces, style): else: raise ValueError("unknown style '%s'" % style) - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} + return { + "version": rendered, + "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], + "error": None, + "date": pieces.get("date"), + } class VersioneerBadRootError(Exception): @@ -1417,8 +1470,9 @@ def get_versions(verbose=False): handlers = HANDLERS.get(cfg.VCS) assert handlers, "unrecognized VCS '%s'" % cfg.VCS verbose = verbose or cfg.verbose - assert cfg.versionfile_source is not None, \ - "please set versioneer.versionfile_source" + assert ( + cfg.versionfile_source is not None + ), "please set versioneer.versionfile_source" assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" versionfile_abs = os.path.join(root, cfg.versionfile_source) @@ -1472,9 +1526,13 @@ def get_versions(verbose=False): if verbose: print("unable to compute version") - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, "error": "unable to compute version", - "date": None} + return { + "version": "0+unknown", + "full-revisionid": None, + "dirty": None, + "error": "unable to compute version", + "date": None, + } def get_version(): @@ -1523,6 +1581,7 @@ def run(self): print(" date: %s" % vers.get("date")) if vers["error"]: print(" error: %s" % vers["error"]) + cmds["version"] = cmd_version # we override "build_py" in both distutils and setuptools @@ -1555,14 +1614,17 @@ def run(self): # now locate _version.py in the new build/ directory and replace # it with an updated value if cfg.versionfile_build: - target_versionfile = os.path.join(self.build_lib, - cfg.versionfile_build) + target_versionfile = os.path.join( + self.build_lib, cfg.versionfile_build + ) print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, versions) + cmds["build_py"] = cmd_build_py if "cx_Freeze" in sys.modules: # cx_freeze enabled? from cx_Freeze.dist import build_exe as _build_exe + # nczeczulin reports that py2exe won't like the pep440-style string # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. # setup(console=[{ @@ -1583,17 +1645,21 @@ def run(self): os.unlink(target_versionfile) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % - {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + cmds["build_exe"] = cmd_build_exe del cmds["build_py"] - if 'py2exe' in sys.modules: # py2exe enabled? + if "py2exe" in sys.modules: # py2exe enabled? try: from py2exe.distutils_buildexe import py2exe as _py2exe # py3 except ImportError: @@ -1612,13 +1678,17 @@ def run(self): os.unlink(target_versionfile) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % - {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + cmds["py2exe"] = cmd_py2exe # we override different "sdist" commands for both environments @@ -1645,8 +1715,10 @@ def make_release_tree(self, base_dir, files): # updated value target_versionfile = os.path.join(base_dir, cfg.versionfile_source) print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, - self._versioneer_generated_versions) + write_to_version_file( + target_versionfile, self._versioneer_generated_versions + ) + cmds["sdist"] = cmd_sdist return cmds @@ -1701,11 +1773,15 @@ def do_setup(): root = get_root() try: cfg = get_config_from_root(root) - except (EnvironmentError, configparser.NoSectionError, - configparser.NoOptionError) as e: + except ( + EnvironmentError, + configparser.NoSectionError, + configparser.NoOptionError, + ) as e: if isinstance(e, (EnvironmentError, configparser.NoSectionError)): - print("Adding sample versioneer config to setup.cfg", - file=sys.stderr) + print( + "Adding sample versioneer config to setup.cfg", file=sys.stderr + ) with open(os.path.join(root, "setup.cfg"), "a") as f: f.write(SAMPLE_CONFIG) print(CONFIG_ERROR, file=sys.stderr) @@ -1714,15 +1790,18 @@ def do_setup(): print(" creating %s" % cfg.versionfile_source) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) - - ipy = os.path.join(os.path.dirname(cfg.versionfile_source), - "__init__.py") + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + + ipy = os.path.join(os.path.dirname(cfg.versionfile_source), "__init__.py") if os.path.exists(ipy): try: with open(ipy, "r") as f: @@ -1764,8 +1843,10 @@ def do_setup(): else: print(" 'versioneer.py' already in MANIFEST.in") if cfg.versionfile_source not in simple_includes: - print(" appending versionfile_source ('%s') to MANIFEST.in" % - cfg.versionfile_source) + print( + " appending versionfile_source ('%s') to MANIFEST.in" + % cfg.versionfile_source + ) with open(manifest_in, "a") as f: f.write("include %s\n" % cfg.versionfile_source) else: From cb6dfdd64b15d0dc85d507576f8b39534ba9af39 Mon Sep 17 00:00:00 2001 From: Robert Lacok Date: Thu, 4 Oct 2018 17:42:38 +0100 Subject: [PATCH 154/519] to_gbq respects location argument properly (#225) If dataset does not exist, it gets created in the correct location --- packages/pandas-gbq/pandas_gbq/gbq.py | 40 +++++++++++++++----- packages/pandas-gbq/tests/system/test_gbq.py | 38 ++++++++++++++++--- 2 files changed, 63 insertions(+), 15 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 79cd1ababf44..5dd6797f3391 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -283,7 +283,7 @@ def __init__( # BQ Queries costs $5 per TB. First 1 TB per month is free # see here for more: https://cloud.google.com/bigquery/pricing - self.query_price_for_TB = 5. / 2 ** 40 # USD/TB + self.query_price_for_TB = 5.0 / 2 ** 40 # USD/TB def _start_timer(self): self.start = time.time() @@ -895,7 +895,11 @@ def to_gbq( dataset_id, table_id = destination_table.rsplit(".", 1) table = _Table( - project_id, dataset_id, reauth=reauth, private_key=private_key + project_id, + dataset_id, + reauth=reauth, + private_key=private_key, + location=location, ) if not table_schema: @@ -967,9 +971,18 @@ def _generate_bq_schema(df, default_type="STRING"): class _Table(GbqConnector): - def __init__(self, project_id, dataset_id, reauth=False, private_key=None): + def __init__( + self, + project_id, + dataset_id, + reauth=False, + private_key=None, + location=None, + ): self.dataset_id = dataset_id - super(_Table, self).__init__(project_id, reauth, private_key) + super(_Table, self).__init__( + project_id, reauth, private_key, location=location + ) def exists(self, table_id): """ Check if a table exists in Google BigQuery @@ -1017,9 +1030,11 @@ def create(self, table_id, schema): if not _Dataset(self.project_id, private_key=self.private_key).exists( self.dataset_id ): - _Dataset(self.project_id, private_key=self.private_key).create( - self.dataset_id - ) + _Dataset( + self.project_id, + private_key=self.private_key, + location=self.location, + ).create(self.dataset_id) table_ref = self.client.dataset(self.dataset_id).table(table_id) table = Table(table_ref) @@ -1064,8 +1079,12 @@ def delete(self, table_id): class _Dataset(GbqConnector): - def __init__(self, project_id, reauth=False, private_key=None): - super(_Dataset, self).__init__(project_id, reauth, private_key) + def __init__( + self, project_id, reauth=False, private_key=None, location=None + ): + super(_Dataset, self).__init__( + project_id, reauth, private_key, location=location + ) def exists(self, dataset_id): """ Check if a dataset exists in Google BigQuery @@ -1107,6 +1126,9 @@ def create(self, dataset_id): dataset = Dataset(self.client.dataset(dataset_id)) + if self.location is not None: + dataset.location = self.location + try: self.client.create_dataset(dataset) except self.http_error as ex: diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index ba85b4c22ea3..c272cae4fd4f 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -741,12 +741,12 @@ def test_query_response_bytes(self): assert self.gbq_connector.sizeof_fmt(1048576) == "1.0 MB" assert self.gbq_connector.sizeof_fmt(1048576000) == "1000.0 MB" assert self.gbq_connector.sizeof_fmt(1073741824) == "1.0 GB" - assert self.gbq_connector.sizeof_fmt(1.099512E12) == "1.0 TB" - assert self.gbq_connector.sizeof_fmt(1.125900E15) == "1.0 PB" - assert self.gbq_connector.sizeof_fmt(1.152922E18) == "1.0 EB" - assert self.gbq_connector.sizeof_fmt(1.180592E21) == "1.0 ZB" - assert self.gbq_connector.sizeof_fmt(1.208926E24) == "1.0 YB" - assert self.gbq_connector.sizeof_fmt(1.208926E28) == "10000.0 YB" + assert self.gbq_connector.sizeof_fmt(1.099512e12) == "1.0 TB" + assert self.gbq_connector.sizeof_fmt(1.125900e15) == "1.0 PB" + assert self.gbq_connector.sizeof_fmt(1.152922e18) == "1.0 EB" + assert self.gbq_connector.sizeof_fmt(1.180592e21) == "1.0 ZB" + assert self.gbq_connector.sizeof_fmt(1.208926e24) == "1.0 YB" + assert self.gbq_connector.sizeof_fmt(1.208926e28) == "10000.0 YB" def test_struct(self, project_id): query = """SELECT 1 int_field, @@ -1325,6 +1325,32 @@ def test_upload_data_tokyo( ) assert table.num_rows > 0 + def test_upload_data_tokyo_non_existing_dataset( + self, project_id, random_dataset_id, bigquery_client + ): + test_size = 10 + df = make_mixed_dataframe_v2(test_size) + non_existing_tokyo_dataset = random_dataset_id + non_existing_tokyo_destination = "{}.to_gbq_test".format( + non_existing_tokyo_dataset + ) + + # Initialize table with sample data + gbq.to_gbq( + df, + non_existing_tokyo_destination, + project_id, + private_key=self.credentials, + location="asia-northeast1", + ) + + table = bigquery_client.get_table( + bigquery_client.dataset(non_existing_tokyo_dataset).table( + "to_gbq_test" + ) + ) + assert table.num_rows > 0 + # _Dataset tests From dbefbac3d94f71e3d75e01e451305282db332092 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Wed, 10 Oct 2018 13:27:45 -0400 Subject: [PATCH 155/519] Parse all date/time types (#224) * parse all datetime types * typo * new black version * I tihnk we're doing similar things twice * better type tests * check nulls before assigning type * add env to gitignore * remove old code * handle float and int columns re nulls * nullable int columns as floats (separate issue) * Chesterton's Fence * try falling back to standard black check * exclude nox * changelog --- packages/pandas-gbq/.gitignore | 1 + packages/pandas-gbq/docs/source/changelog.rst | 6 +++ packages/pandas-gbq/noxfile.py | 8 +--- packages/pandas-gbq/pandas_gbq/gbq.py | 38 ++++++++------- packages/pandas-gbq/pyproject.toml | 1 + packages/pandas-gbq/tests/system/test_gbq.py | 48 +++++++++++-------- packages/pandas-gbq/tests/unit/test_schema.py | 1 - 7 files changed, 57 insertions(+), 46 deletions(-) diff --git a/packages/pandas-gbq/.gitignore b/packages/pandas-gbq/.gitignore index f0dd6fbdeb5e..9fb09906aa4e 100644 --- a/packages/pandas-gbq/.gitignore +++ b/packages/pandas-gbq/.gitignore @@ -22,6 +22,7 @@ .pytest_cache .testmon* .vscode/ +.env # Docs # ######## diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 28cecbcae853..68dc8d606208 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -6,6 +6,10 @@ Changelog 0.7.0 / [unreleased] -------------------- +- `int` columns which contain `NULL` are now cast to `float`, rather than + `object` type. (:issue:`174`) +- `DATE`, `DATETIME` and `TIMESTAMP` columns are now parsed as pandas' `timestamp` + objects (:issue:`224`) - Add :class:`pandas_gbq.Context` to cache credentials in-memory, across calls to ``read_gbq`` and ``to_gbq``. (:issue:`198`, :issue:`208`) - Fast queries now do not log above ``DEBUG`` level. (:issue:`204`) @@ -20,6 +24,8 @@ Internal changes ~~~~~~~~~~~~~~~~ - Avoid listing datasets and tables in system tests. (:issue:`215`) +- Improved performance from eliminating some duplicative parsing steps + (:issue:`224`) .. _changelog-0.6.1: diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 7a76559e26b2..104a34c4a60c 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -77,13 +77,7 @@ def test_latest_deps(session, python=latest_python): @nox.session def lint(session, python=latest_python): session.install("black") - session.run( - "black", - "--check", - "--exclude", - "(\.git|\.hg|\.mypy_cache|\.tox|\.nox|\.venv|_build|buck-out|build|dist)", - ".", - ) + session.run("black", "--check", ".") @nox.session diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 5dd6797f3391..cf57e5746820 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -577,24 +577,41 @@ def _parse_schema(schema_fields): # see: # http://pandas.pydata.org/pandas-docs/dev/missing_data.html # #missing-data-casting-rules-and-indexing - dtype_map = {"FLOAT": np.dtype(float), "TIMESTAMP": "M8[ns]"} + dtype_map = { + "FLOAT": np.dtype(float), + "TIMESTAMP": "datetime64[ns]", + "TIME": "datetime64[ns]", + "DATE": "datetime64[ns]", + "DATETIME": "datetime64[ns]", + "BOOLEAN": bool, + "INTEGER": np.int64, + } for field in schema_fields: name = str(field["name"]) if field["mode"].upper() == "REPEATED": yield name, object else: - dtype = dtype_map.get(field["type"].upper(), object) + dtype = dtype_map.get(field["type"].upper()) yield name, dtype def _parse_data(schema, rows): column_dtypes = OrderedDict(_parse_schema(schema["fields"])) - df = DataFrame(data=(iter(r) for r in rows), columns=column_dtypes.keys()) + for column in df: - df[column] = df[column].astype(column_dtypes[column]) + dtype = column_dtypes[column] + null_safe = ( + df[column].notnull().all() + or dtype == float + or dtype == "datetime64[ns]" + ) + if dtype and null_safe: + df[column] = df[column].astype( + column_dtypes[column], errors="ignore" + ) return df @@ -747,19 +764,6 @@ def read_gbq( "Column order does not match this DataFrame." ) - # cast BOOLEAN and INTEGER columns from object to bool/int - # if they dont have any nulls AND field mode is not repeated (i.e., array) - type_map = {"BOOLEAN": bool, "INTEGER": np.int64} - for field in schema["fields"]: - if ( - field["type"].upper() in type_map - and final_df[field["name"]].notnull().all() - and field["mode"].lower() != "repeated" - ): - final_df[field["name"]] = final_df[field["name"]].astype( - type_map[field["type"].upper()] - ) - connector.log_elapsed_seconds( "Total time taken", datetime.now().strftime("s.\nFinished at %Y-%m-%d %H:%M:%S."), diff --git a/packages/pandas-gbq/pyproject.toml b/packages/pandas-gbq/pyproject.toml index 90440f599070..318a04420365 100644 --- a/packages/pandas-gbq/pyproject.toml +++ b/packages/pandas-gbq/pyproject.toml @@ -4,4 +4,5 @@ exclude = ''' versioneer.py | _version.py | docs +| .nox ''' \ No newline at end of file diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index c272cae4fd4f..51ff11b414f0 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- import sys -from datetime import datetime import uuid +from datetime import datetime import numpy as np import pandas.util.testing as tm @@ -200,9 +200,7 @@ def test_should_properly_handle_nullable_integers(self, project_id): private_key=self.credentials, dialect="legacy", ) - tm.assert_frame_equal( - df, DataFrame({"nullable_integer": [1, None]}).astype(object) - ) + tm.assert_frame_equal(df, DataFrame({"nullable_integer": [1, None]})) def test_should_properly_handle_valid_longs(self, project_id): query = "SELECT 1 << 62 AS valid_long" @@ -225,7 +223,7 @@ def test_should_properly_handle_nullable_longs(self, project_id): dialect="legacy", ) tm.assert_frame_equal( - df, DataFrame({"nullable_long": [1 << 62, None]}).astype(object) + df, DataFrame({"nullable_long": [1 << 62, None]}) ) def test_should_properly_handle_null_integers(self, project_id): @@ -338,35 +336,43 @@ def test_should_properly_handle_arbitrary_timestamp(self, project_id): ), ) - def test_should_properly_handle_null_timestamp(self, project_id): - query = "SELECT TIMESTAMP(NULL) AS null_timestamp" - df = gbq.read_gbq( - query, - project_id=project_id, - private_key=self.credentials, - dialect="legacy", - ) - tm.assert_frame_equal(df, DataFrame({"null_timestamp": [NaT]})) + @pytest.mark.parametrize( + "expression, type_", + [ + ("current_date()", " Date: Fri, 12 Oct 2018 13:14:18 -0500 Subject: [PATCH 156/519] ENH: Catch RefreshError in _try_credentials() (#227) * Catch RefreshError in _try_credentials() * Add to changelog. --- packages/pandas-gbq/docs/source/changelog.rst | 1 + packages/pandas-gbq/pandas_gbq/auth.py | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 68dc8d606208..83c891bc3971 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -19,6 +19,7 @@ Changelog This fixes a bug where pandas-gbq could not refresh credentials if the cached credentials were invalid, revoked, or expired, even when ``reauth=True``. +- Catch RefreshError when trying credentials. (:issue:`226`) Internal changes ~~~~~~~~~~~~~~~~ diff --git a/packages/pandas-gbq/pandas_gbq/auth.py b/packages/pandas-gbq/pandas_gbq/auth.py index 86ad929a54c7..9f32bac77e03 100644 --- a/packages/pandas-gbq/pandas_gbq/auth.py +++ b/packages/pandas-gbq/pandas_gbq/auth.py @@ -297,6 +297,7 @@ def save_user_account_credentials(credentials, credentials_path): def _try_credentials(project_id, credentials): from google.cloud import bigquery import google.api_core.exceptions + import google.auth.exceptions if not credentials: return None @@ -310,3 +311,9 @@ def _try_credentials(project_id, credentials): return credentials except google.api_core.exceptions.GoogleAPIError: return None + except google.auth.exceptions.RefreshError: + # Sometimes (such as on Travis) google-auth returns GCE credentials, + # but fetching the token for those credentials doesn't actually work. + # See: + # https://github.com/googleapis/google-auth-library-python/issues/287 + return None From bfeca69e741ef7eff30c153ccba52009d17c06b9 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 19 Oct 2018 10:28:17 -0700 Subject: [PATCH 157/519] Release 0.7.0 (#229) --- packages/pandas-gbq/docs/source/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 83c891bc3971..dcbb44c4e225 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -3,7 +3,7 @@ Changelog .. _changelog-0.7.0: -0.7.0 / [unreleased] +0.7.0 / 2018-10-19 -------------------- - `int` columns which contain `NULL` are now cast to `float`, rather than From 9f9603100925cb901aa3ab94e02967018d2e232a Mon Sep 17 00:00:00 2001 From: Chris Bandy Date: Fri, 26 Oct 2018 13:12:51 -0500 Subject: [PATCH 158/519] Allow newlines in data passed to to_gbq() (#230) * Allow newlines in data passed to to_gbq() * Add version header to changelog --- packages/pandas-gbq/docs/source/changelog.rst | 7 +++++ packages/pandas-gbq/pandas_gbq/load.py | 1 + packages/pandas-gbq/tests/system/test_gbq.py | 29 +++++++++++++++++++ packages/pandas-gbq/tests/unit/test_load.py | 12 ++++++++ 4 files changed, 49 insertions(+) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index dcbb44c4e225..dff44ef66f41 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,6 +1,13 @@ Changelog ========= +.. _changelog-0.7.1: + +0.7.1 / unreleased +-------------------- + +- Allow newlines in data passed to ``to_gbq``. (:issue:`180`) + .. _changelog-0.7.0: 0.7.0 / 2018-10-19 diff --git a/packages/pandas-gbq/pandas_gbq/load.py b/packages/pandas-gbq/pandas_gbq/load.py index 2cc0accbd817..015f479d16e5 100644 --- a/packages/pandas-gbq/pandas_gbq/load.py +++ b/packages/pandas-gbq/pandas_gbq/load.py @@ -61,6 +61,7 @@ def load_chunks( job_config = bigquery.LoadJobConfig() job_config.write_disposition = "WRITE_APPEND" job_config.source_format = "CSV" + job_config.allow_quoted_newlines = True if schema is None: schema = pandas_gbq.schema.generate_bq_schema(dataframe) diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index 51ff11b414f0..e263f72afe6d 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -1167,6 +1167,35 @@ def test_upload_mixed_float_and_int(self, project_id): assert len(result_df) == test_size + def test_upload_data_with_newlines(self, project_id): + test_id = "data_with_newlines" + test_size = 2 + df = DataFrame({"s": ["abcd", "ef\ngh"]}) + + gbq.to_gbq( + df, + self.destination_table + test_id, + project_id=project_id, + private_key=self.credentials, + ) + + result_df = gbq.read_gbq( + "SELECT * FROM {0}".format(self.destination_table + test_id), + project_id=project_id, + private_key=self.credentials, + dialect="legacy", + ) + + assert len(result_df) == test_size + + if sys.version_info.major < 3: + pytest.skip(msg="Unicode comparison in Py2 not working") + + result = result_df["s"].sort_values() + expected = df["s"].sort_values() + + tm.assert_numpy_array_equal(expected.values, result.values) + def test_upload_data_flexible_column_order(self, project_id): test_id = "13" test_size = 10 diff --git a/packages/pandas-gbq/tests/unit/test_load.py b/packages/pandas-gbq/tests/unit/test_load.py index b53a4dc82203..d2b5860e8246 100644 --- a/packages/pandas-gbq/tests/unit/test_load.py +++ b/packages/pandas-gbq/tests/unit/test_load.py @@ -37,6 +37,18 @@ def test_encode_chunk_with_floats(): assert "1.05153" in csv_string +def test_encode_chunk_with_newlines(): + """See: https://github.com/pydata/pandas-gbq/issues/180 + """ + df = pandas.DataFrame({"s": ["abcd", "ef\ngh", "ij\r\nkl"]}) + csv_buffer = load.encode_chunk(df) + csv_bytes = csv_buffer.read() + csv_string = csv_bytes.decode("utf-8") + assert "abcd" in csv_string + assert '"ef\ngh"' in csv_string + assert '"ij\r\nkl"' in csv_string + + def test_encode_chunks_splits_dataframe(): df = pandas.DataFrame(numpy.random.randn(6, 4), index=range(6)) chunks = list(load.encode_chunks(df, chunksize=2)) From 73f356857edeaf472a8a9a05644e9ffac1e93b98 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 26 Oct 2018 15:02:30 -0700 Subject: [PATCH 159/519] ENH: Add credentials argument to read_gbq and to_gbq. (#231) * ENH: Add credentials argument to read_gbq and to_gbq. Deprecates private_key parameter. * DOC: write files service account example first --- packages/pandas-gbq/docs/source/changelog.rst | 19 +- .../docs/source/howto/authentication.rst | 37 +++- packages/pandas-gbq/pandas_gbq/gbq.py | 95 ++++++--- packages/pandas-gbq/tests/system/test_gbq.py | 180 +++++++++--------- 4 files changed, 213 insertions(+), 118 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index dff44ef66f41..116fa187fc7d 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,11 +1,26 @@ Changelog ========= -.. _changelog-0.7.1: +.. _changelog-0.8.0: -0.7.1 / unreleased +0.8.0 / unreleased -------------------- +Breaking changes +~~~~~~~~~~~~~~~~ + +- **Deprecate** ``private_key`` parameter to :func:`pandas_gbq.read_gbq` and + :func:`pandas_gbq.to_gbq` in favor of new ``credentials`` argument. Instead, + create a credentials object using + :func:`google.oauth2.service_account.Credentials.from_service_account_info` + or + :func:`google.oauth2.service_account.Credentials.from_service_account_file`. + See the :doc:`authentication how-to guide ` for + examples. (:issue:`161`, :issue:`TODO`) + +Enhancements +~~~~~~~~~~~~ + - Allow newlines in data passed to ``to_gbq``. (:issue:`180`) .. _changelog-0.7.0: diff --git a/packages/pandas-gbq/docs/source/howto/authentication.rst b/packages/pandas-gbq/docs/source/howto/authentication.rst index 9f69ed129842..7d54467c3783 100644 --- a/packages/pandas-gbq/docs/source/howto/authentication.rst +++ b/packages/pandas-gbq/docs/source/howto/authentication.rst @@ -18,11 +18,38 @@ Create a service account key via the `service account key creation page the Google Cloud Platform Console. Select the JSON key type and download the key file. -To use service account credentials, set the ``private_key`` parameter to one -of: - -* A file path to the JSON file. -* A string containing the JSON file contents. +To use service account credentials, set the ``credentials`` parameter to the result of a call to: + +* :func:`google.oauth2.service_account.Credentials.from_service_account_file`, + which accepts a file path to the JSON file. + + .. code:: python + + credentials = google.oauth2.service_account.Credentials.from_service_account_file( + 'path/to/key.json', + ) + df = pandas_gbq.read_gbq(sql, project_id="YOUR-PROJECT-ID", credentials=credentials) + +* :func:`google.oauth2.service_account.Credentials.from_service_account_info`, + which accepts a dictionary corresponding to the JSON file contents. + + .. code:: python + + credentials = google.oauth2.service_account.Credentials.from_service_account_info( + { + "type": "service_account", + "project_id": "YOUR-PROJECT-ID", + "private_key_id": "6747200734a1f2b9d8d62fc0b9414c5f2461db0e", + "private_key": "-----BEGIN PRIVATE KEY-----\nM...I==\n-----END PRIVATE KEY-----\n", + "client_email": "service-account@YOUR-PROJECT-ID.iam.gserviceaccount.com", + "client_id": "12345678900001", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://accounts.google.com/o/oauth2/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/...iam.gserviceaccount.com" + }, + ) + df = pandas_gbq.read_gbq(sql, project_id="YOUR-PROJECT-ID", credentials=credentials) See the `Getting started with authentication on Google Cloud Platform `_ guide for diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index cf57e5746820..d31defc0dd20 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -236,6 +236,7 @@ def __init__( dialect="legacy", location=None, try_credentials=None, + credentials=None, ): global context from google.api_core.exceptions import GoogleAPIError @@ -249,11 +250,14 @@ def __init__( self.private_key = private_key self.auth_local_webserver = auth_local_webserver self.dialect = dialect + self.credentials = credentials self.credentials_path = _get_credentials_file() + default_project = None # Load credentials from cache. - self.credentials = context.credentials - default_project = context.project + if not self.credentials: + self.credentials = context.credentials + default_project = context.project # Credentials were explicitly asked for, so don't use the cache. if private_key or reauth or not self.credentials: @@ -563,7 +567,7 @@ def schema_is_subset(self, dataset_id, table_id, schema): def delete_and_recreate_table(self, dataset_id, table_id, table_schema): table = _Table( - self.project_id, dataset_id, private_key=self.private_key + self.project_id, dataset_id, credentials=self.credentials ) table.delete(table_id) table.create(table_id, table_schema) @@ -621,12 +625,13 @@ def read_gbq( index_col=None, col_order=None, reauth=False, - private_key=None, auth_local_webserver=False, dialect=None, location=None, configuration=None, + credentials=None, verbose=None, + private_key=None, ): r"""Load data from Google BigQuery using google-cloud-python @@ -655,10 +660,6 @@ def read_gbq( reauth : boolean, default False Force Google BigQuery to re-authenticate the user. This is useful if multiple accounts are used. - private_key : str, optional - Service account private key in JSON format. Can be file path - or string contents. This is useful for remote server - authentication (eg. Jupyter/IPython notebook on remote host). auth_local_webserver : boolean, default False Use the `local webserver flow`_ instead of the `console flow`_ when getting user credentials. @@ -699,10 +700,28 @@ def read_gbq( For more information see `BigQuery REST API Reference `__. + credentials : google.auth.credentials.Credentials, optional + Credentials for accessing Google APIs. Use this parameter to override + default credentials, such as to use Compute Engine + :class:`google.auth.compute_engine.Credentials` or Service Account + :class:`google.oauth2.service_account.Credentials` directly. + + .. versionadded:: 0.8.0 verbose : None, deprecated Deprecated in Pandas-GBQ 0.4.0. Use the `logging module to adjust verbosity instead `__. + private_key : str, deprecated + Deprecated in pandas-gbq version 0.8.0. Use the ``credentials`` + parameter and + :func:`google.oauth2.service_account.Credentials.from_service_account_info` + or + :func:`google.oauth2.service_account.Credentials.from_service_account_file` + instead. + + Service account private key in JSON format. Can be file path + or string contents. This is useful for remote server + authentication (eg. Jupyter/IPython notebook on remote host). Returns ------- @@ -736,10 +755,11 @@ def read_gbq( connector = GbqConnector( project_id, reauth=reauth, - private_key=private_key, dialect=dialect, auth_local_webserver=auth_local_webserver, location=location, + credentials=credentials, + private_key=private_key, ) schema, rows = connector.run_query(query, configuration=configuration) final_df = _parse_data(schema, rows) @@ -779,12 +799,13 @@ def to_gbq( chunksize=None, reauth=False, if_exists="fail", - private_key=None, auth_local_webserver=False, table_schema=None, location=None, progress_bar=True, + credentials=None, verbose=None, + private_key=None, ): """Write a DataFrame to a Google BigQuery table. @@ -822,10 +843,6 @@ def to_gbq( If table exists, drop it, recreate it, and insert data. ``'append'`` If table exists, insert data. Create if does not exist. - private_key : str, optional - Service account private key in JSON format. Can be file path - or string contents. This is useful for remote server - authentication (eg. Jupyter/IPython notebook on remote host). auth_local_webserver : bool, default False Use the `local webserver flow`_ instead of the `console flow`_ when getting user credentials. @@ -861,10 +878,28 @@ def to_gbq( chunk by chunk. .. versionadded:: 0.5.0 + credentials : google.auth.credentials.Credentials, optional + Credentials for accessing Google APIs. Use this parameter to override + default credentials, such as to use Compute Engine + :class:`google.auth.compute_engine.Credentials` or Service Account + :class:`google.oauth2.service_account.Credentials` directly. + + .. versionadded:: 0.8.0 verbose : bool, deprecated Deprecated in Pandas-GBQ 0.4.0. Use the `logging module to adjust verbosity instead `__. + private_key : str, deprecated + Deprecated in pandas-gbq version 0.8.0. Use the ``credentials`` + parameter and + :func:`google.oauth2.service_account.Credentials.from_service_account_info` + or + :func:`google.oauth2.service_account.Credentials.from_service_account_file` + instead. + + Service account private key in JSON format. Can be file path + or string contents. This is useful for remote server + authentication (eg. Jupyter/IPython notebook on remote host). """ _test_google_api_imports() @@ -889,21 +924,21 @@ def to_gbq( connector = GbqConnector( project_id, reauth=reauth, - private_key=private_key, auth_local_webserver=auth_local_webserver, location=location, # Avoid reads when writing tables. # https://github.com/pydata/pandas-gbq/issues/202 try_credentials=lambda project, creds: creds, + credentials=credentials, + private_key=private_key, ) dataset_id, table_id = destination_table.rsplit(".", 1) table = _Table( project_id, dataset_id, - reauth=reauth, - private_key=private_key, location=location, + credentials=connector.credentials, ) if not table_schema: @@ -980,12 +1015,17 @@ def __init__( project_id, dataset_id, reauth=False, - private_key=None, location=None, + credentials=None, + private_key=None, ): self.dataset_id = dataset_id super(_Table, self).__init__( - project_id, reauth, private_key, location=location + project_id, + reauth, + location=location, + credentials=credentials, + private_key=private_key, ) def exists(self, table_id): @@ -1031,12 +1071,12 @@ def create(self, table_id, schema): "Table {0} already " "exists".format(table_id) ) - if not _Dataset(self.project_id, private_key=self.private_key).exists( + if not _Dataset(self.project_id, credentials=self.credentials).exists( self.dataset_id ): _Dataset( self.project_id, - private_key=self.private_key, + credentials=self.credentials, location=self.location, ).create(self.dataset_id) @@ -1084,10 +1124,19 @@ def delete(self, table_id): class _Dataset(GbqConnector): def __init__( - self, project_id, reauth=False, private_key=None, location=None + self, + project_id, + reauth=False, + location=None, + credentials=None, + private_key=None, ): super(_Dataset, self).__init__( - project_id, reauth, private_key, location=location + project_id, + reauth, + credentials=credentials, + location=location, + private_key=private_key, ) def exists(self, dataset_id): diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index e263f72afe6d..9ae1fbf2f91a 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -4,6 +4,7 @@ import uuid from datetime import datetime +import google.oauth2.service_account import numpy as np import pandas.util.testing as tm import pytest @@ -13,6 +14,7 @@ from pandas_gbq import gbq + TABLE_ID = "new_test" @@ -34,13 +36,15 @@ def project(request, project_id): @pytest.fixture() -def credentials(private_key_contents): - return private_key_contents +def credentials(private_key_path): + return google.oauth2.service_account.Credentials.from_service_account_file( + private_key_path + ) @pytest.fixture() def gbq_connector(project, credentials): - return gbq.GbqConnector(project, private_key=credentials) + return gbq.GbqConnector(project, credentials=credentials) @pytest.fixture(scope="module") @@ -97,12 +101,12 @@ def tokyo_table(bigquery_client, tokyo_dataset): @pytest.fixture() def gbq_dataset(project, credentials): - return gbq._Dataset(project, private_key=credentials) + return gbq._Dataset(project, credentials=credentials) @pytest.fixture() def gbq_table(project, credentials, random_dataset_id): - return gbq._Table(project, random_dataset_id, private_key=credentials) + return gbq._Table(project, random_dataset_id, credentials=credentials) def make_mixed_dataframe_v2(test_size): @@ -146,7 +150,7 @@ def test_should_be_able_to_get_results_from_query(self, gbq_connector): def test_should_read(project, credentials): query = 'SELECT "PI" AS valid_string' df = gbq.read_gbq( - query, project_id=project, private_key=credentials, dialect="legacy" + query, project_id=project, credentials=credentials, dialect="legacy" ) tm.assert_frame_equal(df, DataFrame({"valid_string": ["PI"]})) @@ -157,7 +161,7 @@ def setup(self, project, credentials): # - PER-TEST FIXTURES - # put here any instruction you want to be run *BEFORE* *EVERY* test is # executed. - self.gbq_connector = gbq.GbqConnector(project, private_key=credentials) + self.gbq_connector = gbq.GbqConnector(project, credentials=credentials) self.credentials = credentials def test_should_properly_handle_empty_strings(self, project_id): @@ -165,7 +169,7 @@ def test_should_properly_handle_empty_strings(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal(df, DataFrame({"empty_string": [""]})) @@ -175,7 +179,7 @@ def test_should_properly_handle_null_strings(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal(df, DataFrame({"null_string": [None]})) @@ -185,7 +189,7 @@ def test_should_properly_handle_valid_integers(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal(df, DataFrame({"valid_integer": [3]})) @@ -197,7 +201,7 @@ def test_should_properly_handle_nullable_integers(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal(df, DataFrame({"nullable_integer": [1, None]})) @@ -207,7 +211,7 @@ def test_should_properly_handle_valid_longs(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal(df, DataFrame({"valid_long": [1 << 62]})) @@ -219,7 +223,7 @@ def test_should_properly_handle_nullable_longs(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal( @@ -231,7 +235,7 @@ def test_should_properly_handle_null_integers(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal(df, DataFrame({"null_integer": [None]})) @@ -243,7 +247,7 @@ def test_should_properly_handle_valid_floats(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal(df, DataFrame({"valid_float": [pi]})) @@ -257,7 +261,7 @@ def test_should_properly_handle_nullable_floats(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal(df, DataFrame({"nullable_float": [pi, None]})) @@ -269,7 +273,7 @@ def test_should_properly_handle_valid_doubles(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal( @@ -285,7 +289,7 @@ def test_should_properly_handle_nullable_doubles(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal( @@ -297,7 +301,7 @@ def test_should_properly_handle_null_floats(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal(df, DataFrame({"null_float": [np.nan]})) @@ -307,7 +311,7 @@ def test_should_properly_handle_timestamp_unix_epoch(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal( @@ -322,7 +326,7 @@ def test_should_properly_handle_arbitrary_timestamp(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal( @@ -359,7 +363,7 @@ def test_return_correct_types(self, project_id, expression, type_): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="standard", ) assert df["_"].dtype == type_ @@ -369,7 +373,7 @@ def test_should_properly_handle_null_timestamp(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal(df, DataFrame({"null_timestamp": [NaT]})) @@ -379,7 +383,7 @@ def test_should_properly_handle_null_boolean(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal(df, DataFrame({"null_boolean": [None]})) @@ -391,7 +395,7 @@ def test_should_properly_handle_nullable_booleans(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal( @@ -411,7 +415,7 @@ def test_unicode_string_conversion_and_normalization(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal(df, correct_test_datatype) @@ -422,7 +426,7 @@ def test_index_column(self, project_id): query, project_id=project_id, index_col="string_1", - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) correct_frame = DataFrame( @@ -437,7 +441,7 @@ def test_column_order(self, project_id): query, project_id=project_id, col_order=col_order, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) correct_frame = DataFrame( @@ -455,7 +459,7 @@ def test_read_gbq_raises_invalid_column_order(self, project_id): query, project_id=project_id, col_order=col_order, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) @@ -467,7 +471,7 @@ def test_column_order_plus_index(self, project_id): project_id=project_id, index_col="string_1", col_order=col_order, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) correct_frame = DataFrame( @@ -488,7 +492,7 @@ def test_read_gbq_raises_invalid_index_column(self, project_id): project_id=project_id, index_col="string_bbb", col_order=col_order, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) @@ -497,7 +501,7 @@ def test_malformed_query(self, project_id): gbq.read_gbq( "SELCET * FORM [publicdata:samples.shakespeare]", project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) @@ -506,7 +510,7 @@ def test_bad_project_id(self): gbq.read_gbq( "SELCET * FROM [publicdata:samples.shakespeare]", project_id="not-my-project", - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) @@ -515,7 +519,7 @@ def test_bad_table_name(self, project_id): gbq.read_gbq( "SELECT * FROM [publicdata:samples.nope]", project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) @@ -527,7 +531,7 @@ def test_download_dataset_larger_than_200k_rows(self, project_id): "SELECT id FROM [publicdata:samples.wikipedia] " "GROUP EACH BY id ORDER BY id ASC LIMIT {0}".format(test_size), project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) assert len(df.drop_duplicates()) == test_size @@ -540,7 +544,7 @@ def test_zero_rows(self, project_id): "FROM [publicdata:samples.wikipedia] " "WHERE timestamp=-9999999", project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) page_array = np.zeros( @@ -561,7 +565,7 @@ def test_one_row_one_column(self, project_id): df = gbq.read_gbq( "SELECT 3 as v", project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="standard", ) expected_result = DataFrame(dict(v=[3])) @@ -577,7 +581,7 @@ def test_legacy_sql(self, project_id): legacy_sql, project_id=project_id, dialect="standard", - private_key=self.credentials, + credentials=self.credentials, ) # Test that a legacy sql statement succeeds when @@ -586,7 +590,7 @@ def test_legacy_sql(self, project_id): legacy_sql, project_id=project_id, dialect="legacy", - private_key=self.credentials, + credentials=self.credentials, ) assert len(df.drop_duplicates()) == 10 @@ -602,7 +606,7 @@ def test_standard_sql(self, project_id): gbq.read_gbq( standard_sql, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) @@ -612,7 +616,7 @@ def test_standard_sql(self, project_id): standard_sql, project_id=project_id, dialect="standard", - private_key=self.credentials, + credentials=self.credentials, ) assert len(df.drop_duplicates()) == 10 @@ -642,7 +646,7 @@ def test_query_with_parameters(self, project_id): gbq.read_gbq( sql_statement, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) @@ -651,7 +655,7 @@ def test_query_with_parameters(self, project_id): df = gbq.read_gbq( sql_statement, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, configuration=config, dialect="legacy", ) @@ -667,7 +671,7 @@ def test_query_inside_configuration(self, project_id): gbq.read_gbq( query_no_use, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, configuration=config, dialect="legacy", ) @@ -675,7 +679,7 @@ def test_query_inside_configuration(self, project_id): df = gbq.read_gbq( None, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, configuration=config, dialect="legacy", ) @@ -703,7 +707,7 @@ def test_configuration_without_query(self, project_id): gbq.read_gbq( sql_statement, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, configuration=config, dialect="legacy", ) @@ -721,7 +725,7 @@ def test_configuration_raises_value_error_with_multiple_config( gbq.read_gbq( sql_statement, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, configuration=config, dialect="legacy", ) @@ -734,7 +738,7 @@ def test_timeout_configuration(self, project_id): gbq.read_gbq( sql_statement, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, configuration=config, dialect="legacy", ) @@ -760,7 +764,7 @@ def test_struct(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="standard", ) expected = DataFrame( @@ -774,7 +778,7 @@ def test_array(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="standard", ) tm.assert_frame_equal( @@ -794,7 +798,7 @@ def test_array_length_zero(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="standard", ) expected = DataFrame( @@ -818,7 +822,7 @@ def test_array_agg(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="standard", ) tm.assert_frame_equal( @@ -860,7 +864,7 @@ def setup(self, project, credentials, random_dataset_id): # put here any instruction you want to be run *BEFORE* *EVERY* test is # executed. self.table = gbq._Table( - project, random_dataset_id, private_key=credentials + project, random_dataset_id, credentials=credentials ) self.destination_table = "{}.{}".format(random_dataset_id, TABLE_ID) self.credentials = credentials @@ -875,7 +879,7 @@ def test_upload_data(self, project_id): self.destination_table + test_id, project_id, chunksize=10000, - private_key=self.credentials, + credentials=self.credentials, ) result = gbq.read_gbq( @@ -883,7 +887,7 @@ def test_upload_data(self, project_id): self.destination_table + test_id ), project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) assert result["num_rows"][0] == test_size @@ -900,7 +904,7 @@ def test_upload_data_if_table_exists_fail(self, project_id): df, self.destination_table + test_id, project_id, - private_key=self.credentials, + credentials=self.credentials, ) # Test the if_exists parameter with value 'fail' @@ -910,7 +914,7 @@ def test_upload_data_if_table_exists_fail(self, project_id): self.destination_table + test_id, project_id, if_exists="fail", - private_key=self.credentials, + credentials=self.credentials, ) def test_upload_data_if_table_exists_append(self, project_id): @@ -925,7 +929,7 @@ def test_upload_data_if_table_exists_append(self, project_id): self.destination_table + test_id, project_id, chunksize=10000, - private_key=self.credentials, + credentials=self.credentials, ) # Test the if_exists parameter with value 'append' @@ -934,7 +938,7 @@ def test_upload_data_if_table_exists_append(self, project_id): self.destination_table + test_id, project_id, if_exists="append", - private_key=self.credentials, + credentials=self.credentials, ) result = gbq.read_gbq( @@ -942,7 +946,7 @@ def test_upload_data_if_table_exists_append(self, project_id): self.destination_table + test_id ), project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) assert result["num_rows"][0] == test_size * 2 @@ -954,7 +958,7 @@ def test_upload_data_if_table_exists_append(self, project_id): self.destination_table + test_id, project_id, if_exists="append", - private_key=self.credentials, + credentials=self.credentials, ) def test_upload_subset_columns_if_table_exists_append(self, project_id): @@ -971,7 +975,7 @@ def test_upload_subset_columns_if_table_exists_append(self, project_id): self.destination_table + test_id, project_id, chunksize=10000, - private_key=self.credentials, + credentials=self.credentials, ) # Test the if_exists parameter with value 'append' @@ -980,7 +984,7 @@ def test_upload_subset_columns_if_table_exists_append(self, project_id): self.destination_table + test_id, project_id, if_exists="append", - private_key=self.credentials, + credentials=self.credentials, ) result = gbq.read_gbq( @@ -988,7 +992,7 @@ def test_upload_subset_columns_if_table_exists_append(self, project_id): self.destination_table + test_id ), project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) assert result["num_rows"][0] == test_size * 2 @@ -1005,7 +1009,7 @@ def test_upload_data_if_table_exists_replace(self, project_id): self.destination_table + test_id, project_id, chunksize=10000, - private_key=self.credentials, + credentials=self.credentials, ) # Test the if_exists parameter with the value 'replace'. @@ -1014,7 +1018,7 @@ def test_upload_data_if_table_exists_replace(self, project_id): self.destination_table + test_id, project_id, if_exists="replace", - private_key=self.credentials, + credentials=self.credentials, ) result = gbq.read_gbq( @@ -1022,7 +1026,7 @@ def test_upload_data_if_table_exists_replace(self, project_id): self.destination_table + test_id ), project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) assert result["num_rows"][0] == 5 @@ -1039,7 +1043,7 @@ def test_upload_data_if_table_exists_raises_value_error(self, project_id): self.destination_table + test_id, project_id, if_exists="xxxxx", - private_key=self.credentials, + credentials=self.credentials, ) def test_google_upload_errors_should_raise_exception(self, project_id): @@ -1063,7 +1067,7 @@ def test_google_upload_errors_should_raise_exception(self, project_id): bad_df, self.destination_table + test_id, project_id, - private_key=self.credentials, + credentials=self.credentials, ) def test_upload_chinese_unicode_data(self, project_id): @@ -1078,14 +1082,14 @@ def test_upload_chinese_unicode_data(self, project_id): df, self.destination_table + test_id, project_id, - private_key=self.credentials, + credentials=self.credentials, chunksize=10000, ) result_df = gbq.read_gbq( "SELECT * FROM {0}".format(self.destination_table + test_id), project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) @@ -1118,14 +1122,14 @@ def test_upload_other_unicode_data(self, project_id): df, self.destination_table + test_id, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, chunksize=10000, ) result_df = gbq.read_gbq( "SELECT * FROM {0}".format(self.destination_table + test_id), project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) @@ -1155,13 +1159,13 @@ def test_upload_mixed_float_and_int(self, project_id): df, self.destination_table + test_id, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, ) result_df = gbq.read_gbq( "SELECT * FROM {0}".format(self.destination_table + test_id), project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) @@ -1176,13 +1180,13 @@ def test_upload_data_with_newlines(self, project_id): df, self.destination_table + test_id, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, ) result_df = gbq.read_gbq( "SELECT * FROM {0}".format(self.destination_table + test_id), project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) @@ -1207,7 +1211,7 @@ def test_upload_data_flexible_column_order(self, project_id): self.destination_table + test_id, project_id, chunksize=10000, - private_key=self.credentials, + credentials=self.credentials, ) df_columns_reversed = df[df.columns[::-1]] @@ -1217,7 +1221,7 @@ def test_upload_data_flexible_column_order(self, project_id): self.destination_table + test_id, project_id, if_exists="append", - private_key=self.credentials, + credentials=self.credentials, ) def test_upload_data_with_valid_user_schema(self, project_id): @@ -1236,7 +1240,7 @@ def test_upload_data_with_valid_user_schema(self, project_id): df, destination_table, project_id, - private_key=self.credentials, + credentials=self.credentials, table_schema=test_schema, ) dataset, table = destination_table.split(".") @@ -1261,7 +1265,7 @@ def test_upload_data_with_invalid_user_schema_raises_error( df, destination_table, project_id, - private_key=self.credentials, + credentials=self.credentials, table_schema=test_schema, ) @@ -1281,7 +1285,7 @@ def test_upload_data_with_missing_schema_fields_raises_error( df, destination_table, project_id, - private_key=self.credentials, + credentials=self.credentials, table_schema=test_schema, ) @@ -1299,13 +1303,13 @@ def test_upload_data_with_timestamp(self, project_id): df, self.destination_table + test_id, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, ) result_df = gbq.read_gbq( "SELECT * FROM {0}".format(self.destination_table + test_id), project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) @@ -1331,7 +1335,7 @@ def test_upload_data_with_different_df_and_user_schema(self, project_id): df, destination_table, project_id, - private_key=self.credentials, + credentials=self.credentials, table_schema=test_schema, ) dataset, table = destination_table.split(".") @@ -1351,7 +1355,7 @@ def test_upload_data_tokyo( df, tokyo_destination, project_id, - private_key=self.credentials, + credentials=self.credentials, location="asia-northeast1", ) @@ -1375,7 +1379,7 @@ def test_upload_data_tokyo_non_existing_dataset( df, non_existing_tokyo_destination, project_id, - private_key=self.credentials, + credentials=self.credentials, location="asia-northeast1", ) @@ -1455,7 +1459,7 @@ def test_create_table_data_dataset_does_not_exist( ): table_id = "test_create_table_data_dataset_does_not_exist" table_with_new_dataset = gbq._Table( - project, random_dataset_id, private_key=credentials + project, random_dataset_id, credentials=credentials ) df = make_mixed_dataframe_v2(10) table_with_new_dataset.create(table_id, gbq._generate_bq_schema(df)) From 50bb28399d1cc04ef4fa261548c0d4c9044d4a1c Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 5 Nov 2018 09:44:22 -0800 Subject: [PATCH 160/519] TST: configure CircleCI (#232) Use CircleCI for testing pandas-gbq. This configuration uses the canonical Python and miniconda3 Docker images to reduce problems caused by a unique configuration. Also: * Re-enables conda tests. * Adds Python 3.7 tests. --- packages/pandas-gbq/.circleci/config.yml | 75 +++++++++++++++++++ packages/pandas-gbq/.gitignore | 1 + packages/pandas-gbq/ci/config_auth.sh | 9 +++ .../pandas-gbq/ci/requirements-2.7-0.19.2.pip | 4 - .../pandas-gbq/ci/requirements-3.5-0.18.1.pip | 3 +- .../pandas-gbq/ci/requirements-3.6-MASTER.pip | 8 +- .../pandas-gbq/ci/requirements-3.7-0.23.4.pip | 0 packages/pandas-gbq/ci/run_conda.sh | 27 +++++++ packages/pandas-gbq/ci/run_pip.sh | 21 ++++++ packages/pandas-gbq/ci/run_tests.sh | 11 +++ .../pandas-gbq/docs/source/contributing.rst | 60 ++++++++------- packages/pandas-gbq/tests/system/test_auth.py | 2 +- 12 files changed, 181 insertions(+), 40 deletions(-) create mode 100644 packages/pandas-gbq/.circleci/config.yml create mode 100755 packages/pandas-gbq/ci/config_auth.sh create mode 100644 packages/pandas-gbq/ci/requirements-3.7-0.23.4.pip create mode 100755 packages/pandas-gbq/ci/run_conda.sh create mode 100755 packages/pandas-gbq/ci/run_pip.sh create mode 100755 packages/pandas-gbq/ci/run_tests.sh diff --git a/packages/pandas-gbq/.circleci/config.yml b/packages/pandas-gbq/.circleci/config.yml new file mode 100644 index 000000000000..c0132335dc38 --- /dev/null +++ b/packages/pandas-gbq/.circleci/config.yml @@ -0,0 +1,75 @@ +version: 2 +jobs: + # Pip + "pip-2.7-0.19.2": + docker: + - image: python:2.7 + environment: + PYTHON: "2.7" + PANDAS: "0.19.2" + steps: + - checkout + - run: ci/config_auth.sh + - run: ci/run_pip.sh + "pip-3.5-0.18.1": + docker: + - image: python:3.5 + environment: + PYTHON: "3.5" + PANDAS: "0.18.1" + steps: + - checkout + - run: ci/config_auth.sh + - run: ci/run_pip.sh + "pip-3.6-MASTER": + docker: + - image: python:3.6 + environment: + PYTHON: "3.6" + PANDAS: "MASTER" + steps: + - checkout + - run: ci/config_auth.sh + - run: ci/run_pip.sh + # Coverage + - run: codecov + "pip-3.7-0.23.4": + docker: + - image: python:3.7 + environment: + PYTHON: "3.7" + PANDAS: "0.23.4" + steps: + - checkout + - run: ci/config_auth.sh + - run: ci/run_pip.sh + + # Conda + "conda-3.6-0.20.1": + docker: + - image: continuumio/miniconda3 + environment: + PYTHON: "3.6" + PANDAS: "0.20.1" + steps: + - checkout + - run: ci/config_auth.sh + - run: ci/run_conda.sh + + lint: + docker: + - image: python:3.6 + steps: + - checkout + - run: pip install nox + - run: nox -s lint +workflows: + version: 2 + build: + jobs: + - "pip-2.7-0.19.2" + - "pip-3.5-0.18.1" + - "pip-3.6-MASTER" + - "pip-3.7-0.23.4" + - "conda-3.6-0.20.1" + - lint \ No newline at end of file diff --git a/packages/pandas-gbq/.gitignore b/packages/pandas-gbq/.gitignore index 9fb09906aa4e..d1605d550f42 100644 --- a/packages/pandas-gbq/.gitignore +++ b/packages/pandas-gbq/.gitignore @@ -89,3 +89,4 @@ Thumbs.db # Credentials # ############### bigquery_credentials.dat +ci/service_account.json diff --git a/packages/pandas-gbq/ci/config_auth.sh b/packages/pandas-gbq/ci/config_auth.sh new file mode 100755 index 000000000000..b6e46295dff9 --- /dev/null +++ b/packages/pandas-gbq/ci/config_auth.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -e +# Don't set -x, because we don't want to leak keys. +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" + +# Write key to file if present. +if [ ! -z "$SERVICE_ACCOUNT_KEY" ] ; then + echo "$SERVICE_ACCOUNT_KEY" | base64 --decode > "$DIR"/service_account.json +fi diff --git a/packages/pandas-gbq/ci/requirements-2.7-0.19.2.pip b/packages/pandas-gbq/ci/requirements-2.7-0.19.2.pip index cd94478a457c..932a8957f78a 100644 --- a/packages/pandas-gbq/ci/requirements-2.7-0.19.2.pip +++ b/packages/pandas-gbq/ci/requirements-2.7-0.19.2.pip @@ -1,5 +1 @@ -google-auth -google-auth-oauthlib -PyCrypto mock -google-cloud-bigquery diff --git a/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip b/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip index 116268c7bc1a..618d0ac9144a 100644 --- a/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip +++ b/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip @@ -1,4 +1,3 @@ google-auth==1.4.1 google-auth-oauthlib==0.0.1 -google-cloud-bigquery==0.32.0 -pandas==0.18.1 \ No newline at end of file +google-cloud-bigquery==0.32.0 \ No newline at end of file diff --git a/packages/pandas-gbq/ci/requirements-3.6-MASTER.pip b/packages/pandas-gbq/ci/requirements-3.6-MASTER.pip index bd379b8bbdcb..9ca0b606e608 100644 --- a/packages/pandas-gbq/ci/requirements-3.6-MASTER.pip +++ b/packages/pandas-gbq/ci/requirements-3.6-MASTER.pip @@ -1,5 +1,3 @@ -google-auth -google-auth-oauthlib -git+https://github.com/GoogleCloudPlatform/google-cloud-python.git#egg=version_subpkg&subdirectory=api_core -git+https://github.com/GoogleCloudPlatform/google-cloud-python.git#egg=version_subpkg&subdirectory=core -git+https://github.com/GoogleCloudPlatform/google-cloud-python.git#egg=version_subpkg&subdirectory=bigquery +git+https://github.com/googleapis/google-cloud-python.git#egg=version_subpkg&subdirectory=api_core +git+https://github.com/googleapis/google-cloud-python.git#egg=version_subpkg&subdirectory=core +git+https://github.com/googleapis/google-cloud-python.git#egg=version_subpkg&subdirectory=bigquery diff --git a/packages/pandas-gbq/ci/requirements-3.7-0.23.4.pip b/packages/pandas-gbq/ci/requirements-3.7-0.23.4.pip new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/packages/pandas-gbq/ci/run_conda.sh b/packages/pandas-gbq/ci/run_conda.sh new file mode 100755 index 000000000000..597693286a97 --- /dev/null +++ b/packages/pandas-gbq/ci/run_conda.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -e -x +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" + +# Install dependencies using Conda + +conda config --set always_yes yes --set changeps1 no +conda config --add channels pandas +conda config --add channels conda-forge +conda update -q conda +conda info -a +conda create -q -n test-environment python=$PYTHON +source activate test-environment +if [[ "$PANDAS" == "MASTER" ]]; then + conda install -q numpy pytz python-dateutil; + PRE_WHEELS="https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com"; + pip install --pre --upgrade --timeout=60 -f $PRE_WHEELS pandas; +else + conda install -q pandas=$PANDAS; +fi + +REQ="ci/requirements-${PYTHON}-${PANDAS}" +conda install -q --file "$REQ.conda"; +python setup.py develop + +# Run the tests +$DIR/run_tests.sh diff --git a/packages/pandas-gbq/ci/run_pip.sh b/packages/pandas-gbq/ci/run_pip.sh new file mode 100755 index 000000000000..08628d888ca8 --- /dev/null +++ b/packages/pandas-gbq/ci/run_pip.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -e -x +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" + +# Install dependencies using Pip + +if [[ "$PANDAS" == "MASTER" ]]; then + PRE_WHEELS="https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com"; + pip install --pre --upgrade --timeout=60 -f $PRE_WHEELS pandas; +else + pip install pandas==$PANDAS +fi + +# Install test requirements +pip install coverage pytest pytest-cov flake8 codecov + +REQ="ci/requirements-${PYTHON}-${PANDAS}" +pip install -r "$REQ.pip" +pip install -e . + +$DIR/run_tests.sh diff --git a/packages/pandas-gbq/ci/run_tests.sh b/packages/pandas-gbq/ci/run_tests.sh new file mode 100755 index 000000000000..09a00f7cba03 --- /dev/null +++ b/packages/pandas-gbq/ci/run_tests.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e -x +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" + +if [ -f "$DIR/service_account.json" ]; then + export GOOGLE_APPLICATION_CREDENTIALS="$DIR/service_account.json" +fi + +# Install test requirements +pip install coverage pytest pytest-cov flake8 codecov +pytest -v -m "not local_auth" --cov=pandas_gbq --cov-report xml:/tmp/pytest-cov.xml tests diff --git a/packages/pandas-gbq/docs/source/contributing.rst b/packages/pandas-gbq/docs/source/contributing.rst index 351fccd09962..5ef63dfb7388 100644 --- a/packages/pandas-gbq/docs/source/contributing.rst +++ b/packages/pandas-gbq/docs/source/contributing.rst @@ -105,10 +105,10 @@ want to clone your fork to your machine:: This creates the directory `pandas-gbq-yourname` and connects your repository to the upstream (main project) *pandas-gbq* repository. -The testing suite will run automatically on Travis-CI once your pull request is submitted. +The testing suite will run automatically on CircleCI once your pull request is submitted. However, if you wish to run the test suite on a branch prior to submitting the pull request, -then Travis-CI needs to be hooked up to your GitHub repository. Instructions for doing so -are `here `__. +then CircleCI needs to be hooked up to your GitHub repository. Instructions for doing so +are `here `__. Creating a branch ----------------- @@ -214,11 +214,13 @@ the more common ``PEP8`` issues: - we restrict line-length to 79 characters to promote readability - passing arguments should have spaces after commas, e.g. ``foo(arg1, arg2, kw1='bar')`` -Travis-CI will run the `flake8 `_ tool -and report any stylistic errors in your code. Therefore, it is helpful before -submitting code to run the check yourself on the diff:: +CircleCI will run the `'black' code formatting tool +`_ and report any stylistic errors in your +code. Therefore, it is helpful before submitting code to run the formatter +yourself:: - git diff master | flake8 --diff + pip install black + black . Backwards Compatibility ~~~~~~~~~~~~~~~~~~~~~~~ @@ -287,10 +289,12 @@ directory. Running Google BigQuery Integration Tests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -You will need to create a Google BigQuery private key in JSON format in -order to run Google BigQuery integration tests on your local machine and -on Travis-CI. The first step is to create a `service account -`__. +You will need to create a Google BigQuery private key in JSON format in order +to run Google BigQuery integration tests on your local machine and on +CircleCI. The first step is to create a `service account +`__. Grant the +service account permissions to run BigQuery queries and to create datasets +and tables. To run the integration tests locally, set the following environment variables before running ``pytest``: @@ -301,30 +305,30 @@ before running ``pytest``: Integration tests are skipped in pull requests because the credentials that are required for running Google BigQuery integration tests are -`encrypted `__ -on Travis-CI and are only accessible from the pydata/pandas-gbq repository. The -credentials won't be available on forks of pandas-gbq. Here are the steps to run -gbq integration tests on a forked repository: - -#. Go to `Travis CI `__ and sign in with your GitHub - account. -#. Click on the ``+`` icon next to the ``My Repositories`` list and enable - Travis builds for your fork. -#. Click on the gear icon to edit your travis build, and add two environment +`configured in the CircleCI web interface +`_ +and are only accessible from the pydata/pandas-gbq repository. The +credentials won't be available on forks of pandas-gbq. Here are the steps to +run gbq integration tests on a forked repository: + +#. Go to `CircleCI `__ and sign in with your + GitHub account. +#. Switch to your personal account in the top-left organization switcher. +#. Use the "Add projects" tab to enable CircleCI for your fork. +#. Click on the gear icon to edit your CircleCI build, and add two environment variables: - ``GBQ_PROJECT_ID`` with the value being the ID of your BigQuery project. - - ``SERVICE_ACCOUNT_KEY`` with the value being the *contents* of the JSON - key that you downloaded for your service account. Use single quotes around - your JSON key to ensure that it is treated as a string. + - ``SERVICE_ACCOUNT_KEY`` with the value being the base64-encoded + *contents* of the JSON key that you downloaded for your service account. - For both environment variables, keep the "Display value in build log" option - DISABLED. These variables contain sensitive data and you do not want their - contents being exposed in build logs. + Keep the contents of these variables confidential. These variables contain + sensitive data and you do not want their contents being exposed in build + logs. #. Your branch should be tested automatically once it is pushed. You can check the status by visiting your Travis branches page which exists at the - following location: https://travis-ci.org/your-user-name/pandas-gbq/branches . + following location: https://circleci.com/gh/your-username/pandas-gbq . Click on a build job for your branch. Documenting your code diff --git a/packages/pandas-gbq/tests/system/test_auth.py b/packages/pandas-gbq/tests/system/test_auth.py index f7cdf014769d..c48f45200abc 100644 --- a/packages/pandas-gbq/tests/system/test_auth.py +++ b/packages/pandas-gbq/tests/system/test_auth.py @@ -90,7 +90,7 @@ def test_get_application_default_credentials_returns_credentials(): @pytest.mark.local_auth -def test_get_user_account_credentials_bad_file_returns_credentials(): +def test_get_user_account_credentials_bad_file_returns_user_credentials(): from google.auth.credentials import Credentials with mock.patch("__main__.open", side_effect=IOError()): From 0cef40c2dae4d40c4fff277615ac6bcf02f3c8ea Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Fri, 9 Nov 2018 12:04:46 -0500 Subject: [PATCH 161/519] remove old docs (#233) --- packages/pandas-gbq/docs/source/intro.rst | 17 ----------------- packages/pandas-gbq/docs/source/reading.rst | 3 ++- packages/pandas-gbq/docs/source/tables.rst | 6 ------ packages/pandas-gbq/docs/source/writing.rst | 5 ----- 4 files changed, 2 insertions(+), 29 deletions(-) diff --git a/packages/pandas-gbq/docs/source/intro.rst b/packages/pandas-gbq/docs/source/intro.rst index a7ec65c788ad..49f46614b3c5 100644 --- a/packages/pandas-gbq/docs/source/intro.rst +++ b/packages/pandas-gbq/docs/source/intro.rst @@ -9,23 +9,6 @@ Pandas supports all these `BigQuery data types `__ - -While this trade-off works well for most cases, it breaks down for storing -values greater than 2**53. Such values in BigQuery can represent identifiers -and unnoticed precision lost for identifier is what we want to avoid. - Logging +++++++ diff --git a/packages/pandas-gbq/docs/source/reading.rst b/packages/pandas-gbq/docs/source/reading.rst index b188c39236d4..add61ed27f95 100644 --- a/packages/pandas-gbq/docs/source/reading.rst +++ b/packages/pandas-gbq/docs/source/reading.rst @@ -50,6 +50,7 @@ your job. For more information about query configuration parameters see `here .. note:: The ``dialect`` argument can be used to indicate whether to use BigQuery's ``'legacy'`` SQL - or BigQuery's ``'standard'`` SQL (beta). The default value is ``'legacy'``. For more information + or BigQuery's ``'standard'`` SQL (beta). The default value is ``'legacy'``, though this will change + in a subsequent release to ``'standard'``. For more information on BigQuery's standard SQL, see `BigQuery SQL Reference `__ diff --git a/packages/pandas-gbq/docs/source/tables.rst b/packages/pandas-gbq/docs/source/tables.rst index ba07125d43da..dcf891ee6b1b 100644 --- a/packages/pandas-gbq/docs/source/tables.rst +++ b/packages/pandas-gbq/docs/source/tables.rst @@ -14,9 +14,3 @@ Creating Tables {'name': 'my_int64', 'type': 'INTEGER'}, {'name': 'my_string', 'type': 'STRING'}]} -.. note:: - - If you delete and re-create a BigQuery table with the same name, but different table schema, - you must wait 2 minutes before streaming data into the table. As a workaround, consider creating - the new table with a different name. Refer to - `Google BigQuery issue 191 `__. diff --git a/packages/pandas-gbq/docs/source/writing.rst b/packages/pandas-gbq/docs/source/writing.rst index 14124d03408f..e649d7fdd1de 100644 --- a/packages/pandas-gbq/docs/source/writing.rst +++ b/packages/pandas-gbq/docs/source/writing.rst @@ -49,11 +49,6 @@ a ``TableCreationError`` if the destination table already exists. If an error occurs while streaming data to BigQuery, see `Troubleshooting BigQuery Errors `__. -.. note:: - - The BigQuery SQL query language has some oddities, see the - `BigQuery Query Reference Documentation `__. - .. note:: While BigQuery uses SQL-like syntax, it has some important differences From 1e13ab2a4a225372c4590c93ad32e2c0b5992d9a Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 9 Nov 2018 10:06:54 -0800 Subject: [PATCH 162/519] TST: remove references to Travis. Fix conda tests. (#234) * TST: remove references to Travis. Fix conda tests. Removes the last references to Travis from the README and configuration files. I've attempted to fix the conda build by updating the null floats test to have at neast 1 non-null value. * fix null float test. --- packages/pandas-gbq/.travis.yml | 64 ------------------ packages/pandas-gbq/README.rst | 4 +- packages/pandas-gbq/ci/travis_encrypt_gbq.sh | 34 ---------- packages/pandas-gbq/ci/travis_gbq.json.enc | Bin 2352 -> 0 bytes packages/pandas-gbq/ci/travis_gbq_config.txt | 2 - .../ci/travis_process_gbq_encryption.sh | 13 ---- packages/pandas-gbq/tests/system/conftest.py | 6 +- packages/pandas-gbq/tests/system/test_gbq.py | 8 ++- 8 files changed, 8 insertions(+), 123 deletions(-) delete mode 100644 packages/pandas-gbq/.travis.yml delete mode 100755 packages/pandas-gbq/ci/travis_encrypt_gbq.sh delete mode 100644 packages/pandas-gbq/ci/travis_gbq.json.enc delete mode 100644 packages/pandas-gbq/ci/travis_gbq_config.txt delete mode 100644 packages/pandas-gbq/ci/travis_process_gbq_encryption.sh diff --git a/packages/pandas-gbq/.travis.yml b/packages/pandas-gbq/.travis.yml deleted file mode 100644 index a92996cdb7a9..000000000000 --- a/packages/pandas-gbq/.travis.yml +++ /dev/null @@ -1,64 +0,0 @@ -language: python - -matrix: - include: - - os: linux - python: 2.7 - env: PYTHON=2.7 PANDAS=0.19.2 COVERAGE='false' LINT='false' - - os: linux - python: 3.5 - env: PYTHON=3.5 PANDAS=0.18.1 COVERAGE='true' LINT='false' - # https://github.com/pydata/pandas-gbq/issues/189 - # - os: linux - # python: 3.6 - # env: PYTHON=3.6 PANDAS=0.20.1 COVERAGE='false' LINT='false' - - os: linux - python: 3.6 - env: PYTHON=3.6 PANDAS=MASTER COVERAGE='false' LINT='true' -env: - -before_install: - - echo "before_install" - - source ci/travis_process_gbq_encryption.sh - -install: - # Upgrade setuptools and pip to work around - # https://github.com/pypa/setuptools/issues/885 - - pip install --upgrade setuptools - - pip install --upgrade pip - - REQ="ci/requirements-${PYTHON}-${PANDAS}" ; - if [ -f "$REQ.pip" ]; then - pip install --upgrade nox ; - else - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; - bash miniconda.sh -b -p $HOME/miniconda ; - export PATH="$HOME/miniconda/bin:$PATH" ; - hash -r ; - conda config --set always_yes yes --set changeps1 no ; - conda config --add channels pandas ; - conda config --add channels conda-forge ; - conda update -q conda ; - conda info -a ; - conda create -q -n test-environment python=$PYTHON ; - source activate test-environment ; - conda install -q setuptools ; - conda install -q pandas=$PANDAS; - conda install -q --file "$REQ.conda"; - conda list ; - python setup.py install ; - fi - -script: - - if [[ $PYTHON == '2.7' ]]; then - pip install -r ci/requirements-2.7-0.19.2.pip ; - pip install -e . ; - pytest tests/unit ; - fi - - if [[ $PYTHON == '3.5' ]]; then nox -s test_earliest_deps ; fi - - if [[ $PYTHON == '3.6' ]] && [[ "$PANDAS" == "MASTER" ]]; then nox -s test_latest_deps ; fi - - REQ="ci/requirements-${PYTHON}-${PANDAS}" ; - if [ -f "$REQ.conda" ]; then - pytest --quiet -m 'not local_auth' -v tests ; - fi - - if [[ $COVERAGE == 'true' ]]; then nox -s cover ; fi - - if [[ $LINT == 'true' ]]; then nox -s lint ; fi diff --git a/packages/pandas-gbq/README.rst b/packages/pandas-gbq/README.rst index 87a1cbdce9c7..93c87f39026d 100644 --- a/packages/pandas-gbq/README.rst +++ b/packages/pandas-gbq/README.rst @@ -37,8 +37,8 @@ Usage See the `pandas-gbq documentation `_ for more details. -.. |Build Status| image:: https://travis-ci.org/pydata/pandas-gbq.svg?branch=master - :target: https://travis-ci.org/pydata/pandas-gbq +.. |Build Status| image:: https://circleci.com/gh/pydata/pandas-gbq/tree/master.svg?style=svg + :target: https://circleci.com/gh/pydata/pandas-gbq/tree/master .. |Version Status| image:: https://img.shields.io/pypi/v/pandas-gbq.svg :target: https://pypi.python.org/pypi/pandas-gbq/ .. |Coverage Status| image:: https://img.shields.io/codecov/c/github/pydata/pandas-gbq.svg diff --git a/packages/pandas-gbq/ci/travis_encrypt_gbq.sh b/packages/pandas-gbq/ci/travis_encrypt_gbq.sh deleted file mode 100755 index e404ca73a405..000000000000 --- a/packages/pandas-gbq/ci/travis_encrypt_gbq.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -GBQ_JSON_FILE=$1 - -if [[ $# -ne 1 ]]; then - echo -e "Too few arguments.\nUsage: ./travis_encrypt_gbq.sh "\ - "" - exit 1 -fi - -if [[ $GBQ_JSON_FILE != *.json ]]; then - echo "ERROR: Expected *.json file" - exit 1 -fi - -if [[ ! -f $GBQ_JSON_FILE ]]; then - echo "ERROR: File $GBQ_JSON_FILE does not exist" - exit 1 -fi - -echo "Encrypting $GBQ_JSON_FILE..." -read -d "\n" TRAVIS_KEY TRAVIS_IV <<<$(travis encrypt-file $GBQ_JSON_FILE \ -travis_gbq.json.enc -f | grep -o "\w*_iv\|\w*_key"); - -echo "Adding your secure key to travis_gbq_config.txt ..." -echo -e "TRAVIS_IV_ENV=$TRAVIS_IV\nTRAVIS_KEY_ENV=$TRAVIS_KEY"\ -> travis_gbq_config.txt - -echo "Done. Removing file $GBQ_JSON_FILE" -rm $GBQ_JSON_FILE - -echo -e "Created encrypted credentials file travis_gbq.json.enc.\n"\ - "NOTE: Do NOT commit the *.json file containing your unencrypted" \ - "private key" diff --git a/packages/pandas-gbq/ci/travis_gbq.json.enc b/packages/pandas-gbq/ci/travis_gbq.json.enc deleted file mode 100644 index 6c6735b1a1489832c81393d3c6fb70db9335d2d4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2352 zcmV-03D5TNnqlXdY++)PWyH+_Fy*(K>9equQPR*sFH~Mv7?ngs_L)!gA#S%}I{l{Q zTNi$V9;3O4Z!v+`HNswsC?HD8;K91|kaWxdyIM^>oTz_m-uc5_#E)oKjY~K%0W+_ z{RHA}EvU@I6smY4s&Pv}R!FPGKJm6C)+Xod+@fD<>l7E_I%_#04fxtS<-e6nYZ!A} z_r18*Ck1zY4=na-VJhD!Oov>xZ4w zMPj0)WRya$gv)>0djJ$Y_mhE8yS)5V^`CYksDz)UGTg(^Zo9jeX|+f0RW8w_kDLgL zql4Mht+n8+((UU-@>iV}PLuE}ipFNb$?@5HDvMQQ0@|OAK-2ih|8Ys_@)u@GKc5X~ z3MxX!cC6)~gzV}Rgu@+^I@Kn6huy?c{7cDWS%C`1~T*OxX=SO6Dt*h}~|rwdTcW^0Apns~Sfy6iS$ zzN_)6lJ_KI5?}O4e|i@{bbiRq!mE@On6Y3j(8Mkx54!{5!g3HDE1Cqa#V;VfnRo8?1yB!TduBqgvSB6z@ClrR z;nWlk7X1z#igu=Ch+Z}KD{u7S-<@+N^b_D6t>j2u?>Q127|0797^ce6)}lqHC4cQn zb?WwjDojDqeEBO!M=Xd1K9!tb|3G~&^nN??789{rZH7pcW{b~&DlAE8J5Jm}lvci3 z$SGN%yhSWj2tEzAj&sIlMnTcUmZwsXByIsNBz;gsv%JjwL2{=Z;pUD8bbf~kG{~5S zA2|HLnRKgaSR?fj%>s#ftpcLsE)v#OFX?1Jee*y7YilpKP$K#e;Bes-PIu8~tD>7C zDvc^Gap1e)!A7K)H-k)`1Gk;{WX4f!Kx0l|ms`zT;6_WqBLqgS5j%&b35{rrC9q8R zP-yIY8@Co=8PSojukcIs)Mam^5t@;Pgh(As_!$gF3Ff5+DSq&#_9AHBS{FTSdFIcr zf{dVNI8)^N*8&ru&yT@cm$!_}k;demHQ3I_p$>1tAt+G!EX#ELXgw5ufF_-By!EFj z7-XM|pM!tCin1S-+3vXQGlLfwaQsGOmKGYxA$;L=JrC)xS36iw)5jf_A~eplLwG-g z(X+Do>(x@E1)Q@y(03L}(zkNR>LAUW+|<&{g7E`;TtcB&?s$$=z&EnV# zDn?FMIq>;sGI%E76J)~Nq~|ZyH<+_(4hfXZVWq|j`hrKL@iZUQmk8L%A9(j28ndMg zgL-$qWv>lzw4yT6ED()pFbuzB!f-GL+*ZH7dcUAEaBce1ECbuv4XOu`AQ#RxKj;sO zkrsdRtIpB)(uFI>U_Lq+9GbzW)?j&@9oesUn+?HqT^>@v^aU@j(fD`c8`{i;k?-|} zguupcc_rhf*}ZLM zaBP`mb>AK9l3wYKwHzX!6Ap0A+twAk(iKh5qH;! zm!Cbmr;eY&022xDpsFa#-!uKP(*k^)T`%v;f@2!bV9}qy-N=^+&w;7T`X!IOmbBW@ z^OC_L$GnbnZICluyyKVSfYbLnJ?gQ;o|^SO;TvKq_-)wNnl~jBNKpDorUby$qP+`v z5uMZ{RWGTSuxo5kYHPmAGV34^FF!%@SL^7Q_)ADuLH1U@>EYWjY9a&nq28F5=sza{ z3O?fO^%Q>o<$+EzkI5+6U>bI=RVYekMwJ{j<+-=jnp6+Gx9 zV)Gp@%eX)78ggwV@9%XL&KK;dacY9^-p~5sHM&GQnD=%R?iMt)CmWf>Y$Dy1=Zfvl WtYFY3cqn_L%Flv2;xfP?pVh*qxS|69 diff --git a/packages/pandas-gbq/ci/travis_gbq_config.txt b/packages/pandas-gbq/ci/travis_gbq_config.txt deleted file mode 100644 index fe22dbed8492..000000000000 --- a/packages/pandas-gbq/ci/travis_gbq_config.txt +++ /dev/null @@ -1,2 +0,0 @@ -TRAVIS_IV_ENV=encrypted_53f0a3897064_iv -TRAVIS_KEY_ENV=encrypted_53f0a3897064_key diff --git a/packages/pandas-gbq/ci/travis_process_gbq_encryption.sh b/packages/pandas-gbq/ci/travis_process_gbq_encryption.sh deleted file mode 100644 index 9967d40e49f0..000000000000 --- a/packages/pandas-gbq/ci/travis_process_gbq_encryption.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -source ci/travis_gbq_config.txt - -if [[ -n ${SERVICE_ACCOUNT_KEY} ]]; then - echo "${SERVICE_ACCOUNT_KEY}" > ci/travis_gbq.json; -elif [[ -n ${!TRAVIS_IV_ENV} ]]; then - openssl aes-256-cbc -K ${!TRAVIS_KEY_ENV} -iv ${!TRAVIS_IV_ENV} \ - -in ci/travis_gbq.json.enc -out ci/travis_gbq.json -d; - export GBQ_PROJECT_ID='pandas-travis'; - echo 'Successfully decrypted gbq credentials' -fi - diff --git a/packages/pandas-gbq/tests/system/conftest.py b/packages/pandas-gbq/tests/system/conftest.py index cd5a89be6b65..5091d71725d5 100644 --- a/packages/pandas-gbq/tests/system/conftest.py +++ b/packages/pandas-gbq/tests/system/conftest.py @@ -16,11 +16,7 @@ def project_id(): @pytest.fixture(scope="session") def private_key_path(): path = None - if "TRAVIS_BUILD_DIR" in os.environ: - path = os.path.join( - os.environ["TRAVIS_BUILD_DIR"], "ci", "travis_gbq.json" - ) - elif "GBQ_GOOGLE_APPLICATION_CREDENTIALS" in os.environ: + if "GBQ_GOOGLE_APPLICATION_CREDENTIALS" in os.environ: path = os.environ["GBQ_GOOGLE_APPLICATION_CREDENTIALS"] elif "GOOGLE_APPLICATION_CREDENTIALS" in os.environ: path = os.environ["GOOGLE_APPLICATION_CREDENTIALS"] diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index 9ae1fbf2f91a..dde34cb139e0 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -297,14 +297,16 @@ def test_should_properly_handle_nullable_doubles(self, project_id): ) def test_should_properly_handle_null_floats(self, project_id): - query = "SELECT FLOAT(NULL) AS null_float" + query = """SELECT null_float + FROM UNNEST(ARRAY[NULL, 1.0]) AS null_float + """ df = gbq.read_gbq( query, project_id=project_id, credentials=self.credentials, - dialect="legacy", + dialect="standard", ) - tm.assert_frame_equal(df, DataFrame({"null_float": [np.nan]})) + tm.assert_frame_equal(df, DataFrame({"null_float": [np.nan, 1.0]})) def test_should_properly_handle_timestamp_unix_epoch(self, project_id): query = 'SELECT TIMESTAMP("1970-01-01 00:00:00") AS unix_epoch' From 180f2a6c6fe8696cadbe3bf07ad392beec40d62f Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 9 Nov 2018 15:16:51 -0800 Subject: [PATCH 163/519] ENH: Add pandas_gbq.context.dialect to modify default dialect (#235) This will allow people to revert the dialect to legacy SQL when we switch the default or use standard SQL as the default earlier. --- packages/pandas-gbq/pandas_gbq/gbq.py | 33 +++++++++++++++++-- .../pandas-gbq/tests/unit/test_context.py | 18 ++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index d31defc0dd20..13d4bbfd15e2 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -170,6 +170,8 @@ class Context(object): def __init__(self): self._credentials = None self._project = None + # dialect defaults to None so that read_gbq can stop warning if set. + self._dialect = None @property def credentials(self): @@ -216,6 +218,28 @@ def project(self): def project(self, value): self._project = value + @property + def dialect(self): + """str: Default dialect to use in :func:`pandas_gbq.read_gbq`. + + Allowed values for the BigQuery SQL syntax dialect: + + ``'legacy'`` + Use BigQuery's legacy SQL dialect. For more information see + `BigQuery Legacy SQL Reference + `__. + ``'standard'`` + Use BigQuery's standard SQL, which is + compliant with the SQL 2011 standard. For more information + see `BigQuery Standard SQL Reference + `__. + """ + return self._dialect + + @dialect.setter + def dialect(self, value): + self._dialect = value + # Create an empty context, used to cache credentials. context = Context() @@ -728,12 +752,17 @@ def read_gbq( df: DataFrame DataFrame representing results of query. """ + global context + + if dialect is None: + dialect = context.dialect + if dialect is None: dialect = "legacy" warnings.warn( 'The default value for dialect is changing to "standard" in a ' - 'future version. Pass in dialect="legacy" to disable this ' - "warning.", + 'future version. Pass in dialect="legacy" or set ' + 'pandas_gbq.context.dialect="legacy" to disable this warning.', FutureWarning, stacklevel=2, ) diff --git a/packages/pandas-gbq/tests/unit/test_context.py b/packages/pandas-gbq/tests/unit/test_context.py index 352ece7ecd09..ffb6a4e7480e 100644 --- a/packages/pandas-gbq/tests/unit/test_context.py +++ b/packages/pandas-gbq/tests/unit/test_context.py @@ -36,3 +36,21 @@ def test_read_gbq_should_save_credentials(mock_get_credentials): pandas_gbq.read_gbq("SELECT 1", dialect="standard") mock_get_credentials.assert_not_called() + + +def test_read_gbq_should_use_dialect(mock_bigquery_client): + import pandas_gbq + + assert pandas_gbq.context.dialect is None + pandas_gbq.context.dialect = "legacy" + pandas_gbq.read_gbq("SELECT 1") + + _, kwargs = mock_bigquery_client.query.call_args + assert kwargs["job_config"].use_legacy_sql == True + + pandas_gbq.context.dialect = "standard" + pandas_gbq.read_gbq("SELECT 1") + + _, kwargs = mock_bigquery_client.query.call_args + assert kwargs["job_config"].use_legacy_sql == False + pandas_gbq.context.dialect = None # Reset the global state. From 5d63e7c84c315fcf762fc654540c15728c9b6d87 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 12 Nov 2018 01:05:05 -0800 Subject: [PATCH 164/519] Release 0.8.0 (#236) * Release 0.8.0 * Update changelog. * Add Python 3.7 to supported versions in the package. * Fix docstrings in new Context class to match pandas-style. * Document new context class in authentication and API docs. * Add issue number for `credentials` parameter change. --- packages/pandas-gbq/docs/source/api.rst | 4 + packages/pandas-gbq/docs/source/changelog.rst | 14 +++- .../pandas-gbq/docs/source/contributing.rst | 2 +- .../docs/source/howto/authentication.rst | 12 ++- packages/pandas-gbq/pandas_gbq/gbq.py | 77 +++++++++++++------ packages/pandas-gbq/setup.py | 1 + 6 files changed, 77 insertions(+), 33 deletions(-) diff --git a/packages/pandas-gbq/docs/source/api.rst b/packages/pandas-gbq/docs/source/api.rst index a189bae577ba..d0b6622158cd 100644 --- a/packages/pandas-gbq/docs/source/api.rst +++ b/packages/pandas-gbq/docs/source/api.rst @@ -18,6 +18,10 @@ API Reference Context .. autofunction:: read_gbq + .. autofunction:: to_gbq + .. autodata:: context + .. autoclass:: Context + :members: diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 116fa187fc7d..732be6185984 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -3,8 +3,8 @@ Changelog .. _changelog-0.8.0: -0.8.0 / unreleased --------------------- +0.8.0 / 2018-11-12 +------------------ Breaking changes ~~~~~~~~~~~~~~~~ @@ -16,12 +16,20 @@ Breaking changes or :func:`google.oauth2.service_account.Credentials.from_service_account_file`. See the :doc:`authentication how-to guide ` for - examples. (:issue:`161`, :issue:`TODO`) + examples. (:issue:`161`, :issue:`231`) Enhancements ~~~~~~~~~~~~ - Allow newlines in data passed to ``to_gbq``. (:issue:`180`) +- Add :attr:`pandas_gbq.context.dialect` to allow overriding the default SQL + syntax dialect. (:issue:`195`, :issue:`235`) +- Support Python 3.7. (:issue:`197`, :issue:`232`) + +Internal changes +~~~~~~~~~~~~~~~~ + +- Migrate tests to CircleCI. (:issue:`228`, :issue:`232`) .. _changelog-0.7.0: diff --git a/packages/pandas-gbq/docs/source/contributing.rst b/packages/pandas-gbq/docs/source/contributing.rst index 5ef63dfb7388..cacbf1c4d6c5 100644 --- a/packages/pandas-gbq/docs/source/contributing.rst +++ b/packages/pandas-gbq/docs/source/contributing.rst @@ -321,7 +321,7 @@ run gbq integration tests on a forked repository: - ``GBQ_PROJECT_ID`` with the value being the ID of your BigQuery project. - ``SERVICE_ACCOUNT_KEY`` with the value being the base64-encoded - *contents* of the JSON key that you downloaded for your service account. + *contents* of the JSON key that you downloaded for your service account. Keep the contents of these variables confidential. These variables contain sensitive data and you do not want their contents being exposed in build diff --git a/packages/pandas-gbq/docs/source/howto/authentication.rst b/packages/pandas-gbq/docs/source/howto/authentication.rst index 7d54467c3783..e75e87778ab8 100644 --- a/packages/pandas-gbq/docs/source/howto/authentication.rst +++ b/packages/pandas-gbq/docs/source/howto/authentication.rst @@ -58,10 +58,14 @@ more information on service accounts. Default Authentication Methods ------------------------------ -If the ``private_key`` parameter is ``None``, pandas-gbq tries the following -authentication methods: +If the ``credentials`` parameter (or the deprecated ``private_key`` +parameter) is ``None``, pandas-gbq tries the following authentication +methods: -1. Application Default Credentials via the :func:`google.auth.default` +1. In-memory, cached credentials at ``pandas_gbq.context.credentials``. See + :attr:`pandas_gbq.Context.credentials` for details. + +2. Application Default Credentials via the :func:`google.auth.default` function. .. note:: @@ -74,7 +78,7 @@ authentication methods: Compute Engine is that the VM does not have sufficient scopes to query BigQuery. -2. User account credentials. +3. User account credentials. pandas-gbq loads cached credentials from a hidden user folder on the operating system. Override the location of the cached user credentials diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 13d4bbfd15e2..bf02bdad9aa7 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -175,27 +175,34 @@ def __init__(self): @property def credentials(self): - """google.auth.credentials.Credentials: Credentials to use for Google - APIs. - - Note: - These credentials are automatically cached in memory by calls to - :func:`pandas_gbq.read_gbq` and :func:`pandas_gbq.to_gbq`. To - manually set the credentials, construct an - :class:`google.auth.credentials.Credentials` object and set it as - the context credentials as demonstrated in the example below. See - `auth docs`_ for more information on obtaining credentials. - - Example: - Manually setting the context credentials: - >>> import pandas_gbq - >>> from google.oauth2 import service_account - >>> credentials = (service_account - ... .Credentials.from_service_account_file( - ... '/path/to/key.json')) - >>> pandas_gbq.context.credentials = credentials + """ + Credentials to use for Google APIs. + + These credentials are automatically cached in memory by calls to + :func:`pandas_gbq.read_gbq` and :func:`pandas_gbq.to_gbq`. To + manually set the credentials, construct an + :class:`google.auth.credentials.Credentials` object and set it as + the context credentials as demonstrated in the example below. See + `auth docs`_ for more information on obtaining credentials. + .. _auth docs: http://google-auth.readthedocs.io /en/latest/user-guide.html#obtaining-credentials + + Returns + ------- + google.auth.credentials.Credentials + + Examples + -------- + + Manually setting the context credentials: + + >>> import pandas_gbq + >>> from google.oauth2 import service_account + >>> credentials = service_account.Credentials.from_service_account_file( + ... '/path/to/key.json', + ... ) + >>> pandas_gbq.context.credentials = credentials """ return self._credentials @@ -205,12 +212,19 @@ def credentials(self, value): @property def project(self): - """str: Default project to use for calls to Google APIs. + """Default project to use for calls to Google APIs. - Example: - Manually setting the context project: - >>> import pandas_gbq - >>> pandas_gbq.context.project = 'my-project' + Returns + ------- + str + + Examples + -------- + + Manually setting the context project: + + >>> import pandas_gbq + >>> pandas_gbq.context.project = 'my-project' """ return self._project @@ -220,7 +234,8 @@ def project(self, value): @property def dialect(self): - """str: Default dialect to use in :func:`pandas_gbq.read_gbq`. + """ + Default dialect to use in :func:`pandas_gbq.read_gbq`. Allowed values for the BigQuery SQL syntax dialect: @@ -233,6 +248,18 @@ def dialect(self): compliant with the SQL 2011 standard. For more information see `BigQuery Standard SQL Reference `__. + + Returns + ------- + str + + Examples + -------- + + Setting the default syntax to standard: + + >>> import pandas_gbq + >>> pandas_gbq.context.dialect = 'standard' """ return self._dialect diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index 61eb8691f4c9..bd0c0d1184c8 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -47,6 +47,7 @@ def readme(): "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", "Topic :: Scientific/Engineering", ], keywords="data", From c61e205e172b98e9defdc515f00622a0af7d1e7c Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 19 Dec 2018 17:44:17 -0800 Subject: [PATCH 165/519] ENH: deprecate private_key argument (#240) The argument was already deprecated in the function docs. This commit adds a warning when using pandas 0.24 or greater (the first version to include the credentials argument). --- .../docs/source/howto/authentication.rst | 10 +- packages/pandas-gbq/pandas_gbq/gbq.py | 23 ++++- packages/pandas-gbq/tests/unit/test_gbq.py | 93 +++++++++++++++++++ 3 files changed, 123 insertions(+), 3 deletions(-) diff --git a/packages/pandas-gbq/docs/source/howto/authentication.rst b/packages/pandas-gbq/docs/source/howto/authentication.rst index e75e87778ab8..a44a61c78611 100644 --- a/packages/pandas-gbq/docs/source/howto/authentication.rst +++ b/packages/pandas-gbq/docs/source/howto/authentication.rst @@ -25,7 +25,10 @@ To use service account credentials, set the ``credentials`` parameter to the res .. code:: python - credentials = google.oauth2.service_account.Credentials.from_service_account_file( + from google.oauth2 import service_account + import pandas_gbq + + credentials = service_account.Credentials.from_service_account_file( 'path/to/key.json', ) df = pandas_gbq.read_gbq(sql, project_id="YOUR-PROJECT-ID", credentials=credentials) @@ -35,7 +38,10 @@ To use service account credentials, set the ``credentials`` parameter to the res .. code:: python - credentials = google.oauth2.service_account.Credentials.from_service_account_info( + from google.oauth2 import service_account + import pandas_gbq + + credentials = service_account.Credentials.from_service_account_info( { "type": "service_account", "project_id": "YOUR-PROJECT-ID", diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index bf02bdad9aa7..c71b6cc5d192 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -14,6 +14,13 @@ BIGQUERY_INSTALLED_VERSION = None SHOW_VERBOSE_DEPRECATION = False +SHOW_PRIVATE_KEY_DEPRECATION = False +PRIVATE_KEY_DEPRECATION_MESSAGE = ( + "private_key is deprecated and will be removed in a future version." + "Use the credentials argument instead. See " + "https://pandas-gbq.readthedocs.io/en/latest/howto/authentication.html " + "for examples on using the credentials argument with service account keys." +) try: import tqdm # noqa @@ -22,7 +29,7 @@ def _check_google_client_version(): - global BIGQUERY_INSTALLED_VERSION, SHOW_VERBOSE_DEPRECATION + global BIGQUERY_INSTALLED_VERSION, SHOW_VERBOSE_DEPRECATION, SHOW_PRIVATE_KEY_DEPRECATION try: import pkg_resources @@ -53,6 +60,10 @@ def _check_google_client_version(): SHOW_VERBOSE_DEPRECATION = ( pandas_installed_version >= pandas_version_wo_verbosity ) + pandas_version_with_credentials_arg = pkg_resources.parse_version("0.24.0") + SHOW_PRIVATE_KEY_DEPRECATION = ( + pandas_installed_version >= pandas_version_with_credentials_arg + ) def _test_google_api_imports(): @@ -805,6 +816,11 @@ def read_gbq( stacklevel=2, ) + if private_key is not None and SHOW_PRIVATE_KEY_DEPRECATION: + warnings.warn( + PRIVATE_KEY_DEPRECATION_MESSAGE, FutureWarning, stacklevel=2 + ) + if dialect not in ("legacy", "standard"): raise ValueError("'{0}' is not valid for dialect".format(dialect)) @@ -969,6 +985,11 @@ def to_gbq( stacklevel=1, ) + if private_key is not None and SHOW_PRIVATE_KEY_DEPRECATION: + warnings.warn( + PRIVATE_KEY_DEPRECATION_MESSAGE, FutureWarning, stacklevel=2 + ) + if if_exists not in ("fail", "replace", "append"): raise ValueError("'{0}' is not valid for if_exists".format(if_exists)) diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index 1f3ec9a4fd43..004360288e94 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -187,6 +187,57 @@ def test_to_gbq_with_verbose_old_pandas_no_warnings(recwarn, min_bq_version): assert len(recwarn) == 0 +def test_to_gbq_with_private_key_new_pandas_warns_deprecation( + min_bq_version, monkeypatch +): + import pkg_resources + from pandas_gbq import auth + + monkeypatch.setattr(auth, "get_credentials", mock_get_credentials) + + pandas_version = pkg_resources.parse_version("0.24.0") + with pytest.warns(FutureWarning), mock.patch( + "pkg_resources.Distribution.parsed_version", + new_callable=mock.PropertyMock, + ) as mock_version: + mock_version.side_effect = [min_bq_version, pandas_version] + try: + gbq.to_gbq( + DataFrame([[1]]), + "dataset.tablename", + project_id="my-project", + private_key="path/to/key.json", + ) + except gbq.TableCreationError: + pass + + +def test_to_gbq_with_private_key_old_pandas_no_warnings( + recwarn, min_bq_version, monkeypatch +): + import pkg_resources + from pandas_gbq import auth + + monkeypatch.setattr(auth, "get_credentials", mock_get_credentials) + + pandas_version = pkg_resources.parse_version("0.23.4") + with mock.patch( + "pkg_resources.Distribution.parsed_version", + new_callable=mock.PropertyMock, + ) as mock_version: + mock_version.side_effect = [min_bq_version, pandas_version] + try: + gbq.to_gbq( + DataFrame([[1]]), + "dataset.tablename", + project_id="my-project", + private_key="path/to/key.json", + ) + except gbq.TableCreationError: + pass + assert len(recwarn) == 0 + + def test_to_gbq_doesnt_run_query( recwarn, mock_bigquery_client, min_bq_version ): @@ -334,6 +385,48 @@ def test_read_gbq_with_verbose_old_pandas_no_warnings(recwarn, min_bq_version): assert len(recwarn) == 0 +def test_read_gbq_with_private_key_new_pandas_warns_deprecation( + min_bq_version, monkeypatch +): + import pkg_resources + from pandas_gbq import auth + + monkeypatch.setattr(auth, "get_credentials", mock_get_credentials) + + pandas_version = pkg_resources.parse_version("0.24.0") + with pytest.warns(FutureWarning), mock.patch( + "pkg_resources.Distribution.parsed_version", + new_callable=mock.PropertyMock, + ) as mock_version: + mock_version.side_effect = [min_bq_version, pandas_version] + gbq.read_gbq( + "SELECT 1", project_id="my-project", private_key="path/to/key.json" + ) + + +def test_read_gbq_with_private_key_old_pandas_no_warnings( + recwarn, min_bq_version, monkeypatch +): + import pkg_resources + from pandas_gbq import auth + + monkeypatch.setattr(auth, "get_credentials", mock_get_credentials) + + pandas_version = pkg_resources.parse_version("0.23.4") + with mock.patch( + "pkg_resources.Distribution.parsed_version", + new_callable=mock.PropertyMock, + ) as mock_version: + mock_version.side_effect = [min_bq_version, pandas_version] + gbq.read_gbq( + "SELECT 1", + project_id="my-project", + dialect="standard", + private_key="path/to/key.json", + ) + assert len(recwarn) == 0 + + def test_read_gbq_with_invalid_dialect(): with pytest.raises(ValueError) as excinfo: gbq.read_gbq("SELECT 1", dialect="invalid") From 9809b61079963be39de6fb20eb4d728e746b23dc Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 4 Jan 2019 09:18:05 -0800 Subject: [PATCH 166/519] CLN: use pydata-google-auth for auth flow (#241) * CLN: use pydata-google-auth for auth flow Only private_key logic and customized path for credentials cache remain. At some point in the future private_key logic will be removed, as that parameter is deprecated in favor of the credentials argument. Also removes the _try_credentials logic, as that slows down the authentication process and is largely unnecessary now that credentials can be explicitly created and supplied via the credentials argument. * Add user auth detail to authentication guide. * Add comment explaining client_id and client_secret. --- packages/pandas-gbq/docs/source/changelog.rst | 11 + packages/pandas-gbq/docs/source/conf.py | 1 + .../docs/source/howto/authentication.rst | 99 +++++- packages/pandas-gbq/docs/source/install.rst | 1 + packages/pandas-gbq/pandas_gbq/auth.py | 281 +++--------------- packages/pandas-gbq/pandas_gbq/gbq.py | 17 +- packages/pandas-gbq/setup.py | 1 + packages/pandas-gbq/tests/system/test_auth.py | 106 +++---- packages/pandas-gbq/tests/unit/test_auth.py | 13 +- packages/pandas-gbq/tests/unit/test_gbq.py | 34 +-- 10 files changed, 223 insertions(+), 341 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 732be6185984..961f289c7f45 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,6 +1,17 @@ Changelog ========= +.. _changelog-0.9.0: + +0.9.0 / TBD +----------- + +Internal changes +~~~~~~~~~~~~~~~~ + +- **New dependency** Use the ``pydata-google-auth`` package for + authentication. (:issue:`241`) + .. _changelog-0.8.0: 0.8.0 / 2018-11-12 diff --git a/packages/pandas-gbq/docs/source/conf.py b/packages/pandas-gbq/docs/source/conf.py index 40c3911443f2..1959fc36b6f8 100644 --- a/packages/pandas-gbq/docs/source/conf.py +++ b/packages/pandas-gbq/docs/source/conf.py @@ -372,6 +372,7 @@ intersphinx_mapping = { "https://docs.python.org/": None, "https://pandas.pydata.org/pandas-docs/stable/": None, + "https://pydata-google-auth.readthedocs.io/en/latest/": None, "https://google-auth.readthedocs.io/en/latest/": None, } diff --git a/packages/pandas-gbq/docs/source/howto/authentication.rst b/packages/pandas-gbq/docs/source/howto/authentication.rst index a44a61c78611..4612bc4bb0b9 100644 --- a/packages/pandas-gbq/docs/source/howto/authentication.rst +++ b/packages/pandas-gbq/docs/source/howto/authentication.rst @@ -7,7 +7,7 @@ pandas-gbq `authenticates with the Google BigQuery service .. _authentication: -Authentication with a Service Account +Authenticating with a Service Account -------------------------------------- Using service account credentials is particularly useful when working on @@ -57,10 +57,81 @@ To use service account credentials, set the ``credentials`` parameter to the res ) df = pandas_gbq.read_gbq(sql, project_id="YOUR-PROJECT-ID", credentials=credentials) +Use the :func:`~google.oauth2.service_account.Credentials.with_scopes` method +to use authorize with specific OAuth2 scopes, which may be required in +queries to federated data sources such as Google Sheets. + +.. code:: python + + credentials = ... + credentials = credentials.with_scopes( + [ + 'https://www.googleapis.com/auth/drive', + 'https://www.googleapis.com/auth/cloud-platform', + ], + ) + df = pandas_gbq.read_gbq(..., credentials=credentials) + See the `Getting started with authentication on Google Cloud Platform `_ guide for more information on service accounts. + +Authenticating with a User Account +---------------------------------- + +Use the `pydata-google-auth `__ +library to authenticate with a user account (i.e. a G Suite or Gmail +account). The :func:`pydata_google_auth.get_user_credentials` function loads +credentials from a cache on disk or initiates an OAuth 2.0 flow if cached +credentials are not found. + +.. code:: python + + import pandas_gbq + import pydata_google_auth + + SCOPES = [ + 'https://www.googleapis.com/auth/cloud-platform', + 'https://www.googleapis.com/auth/drive', + ] + + credentials = pydata_google_auth.get_user_credentials( + SCOPES, + # Set auth_local_webserver to True to have a slightly more convienient + # authorization flow. Note, this doesn't work if you're running from a + # notebook on a remote sever, such as over SSH or with Google Colab. + auth_local_webserver=True, + + + df = pandas_gbq.read_gbq( + "SELECT my_col FROM `my_dataset.my_table`", + project_id='YOUR-PROJECT-ID', + credentials=credentials, + ) + +.. warning:: + + Do not store credentials on disk when using shared computing resources + such as a GCE VM or Colab notebook. Use the + :data:`pydata_google_auth.cache.NOOP` cache to avoid writing credentials + to disk. + + .. code:: python + + import pydata_google_auth.cache + + credentials = pydata_google_auth.get_user_credentials( + SCOPES, + # Use the NOOP cache to avoid writing credentials to disk. + cache=pydata_google_auth.cache.NOOP, + ) + +Additional information on the user credentials authentication mechanism +can be found in the `Google Cloud authentication guide +`__. + + Default Authentication Methods ------------------------------ @@ -71,6 +142,19 @@ methods: 1. In-memory, cached credentials at ``pandas_gbq.context.credentials``. See :attr:`pandas_gbq.Context.credentials` for details. + .. code:: python + + import pandas_gbq + + credentials = ... # From google-auth or pydata-google-auth library. + + # Update the in-memory credentials cache (added in pandas-gbq 0.7.0). + pandas_gbq.context.credentials = credentials + pandas_gbq.context.project = "your-project-id" + + # The credentials and project_id arguments can be omitted. + df = pandas_gbq.read_gbq("SELECT my_col FROM `my_dataset.my_table`") + 2. Application Default Credentials via the :func:`google.auth.default` function. @@ -87,13 +171,14 @@ methods: 3. User account credentials. pandas-gbq loads cached credentials from a hidden user folder on the - operating system. Override the location of the cached user credentials - by setting the ``PANDAS_GBQ_CREDENTIALS_FILE`` environment variable. + operating system. + + Windows + ``%APPDATA%\pandas_gbq\bigquery_credentials.dat`` + + Linux/Mac/Unix + ``~/.config/pandas_gbq/bigquery_credentials.dat`` If pandas-gbq does not find cached credentials, it opens a browser window asking for you to authenticate to your BigQuery account using the product name ``pandas GBQ``. - - Additional information on the user credentails authentication mechanism - can be found `here - `__. diff --git a/packages/pandas-gbq/docs/source/install.rst b/packages/pandas-gbq/docs/source/install.rst index c64c7939d24f..457a33e9ae85 100644 --- a/packages/pandas-gbq/docs/source/install.rst +++ b/packages/pandas-gbq/docs/source/install.rst @@ -37,6 +37,7 @@ Dependencies This module requires following additional dependencies: +- `pydata-google-auth `__: Helpers for authentication to Google's API - `google-auth `__: authentication and authorization for Google's API - `google-auth-oauthlib `__: integration with `oauthlib `__ for end-user authentication - `google-cloud-bigquery `__: Google Cloud client library for BigQuery diff --git a/packages/pandas-gbq/pandas_gbq/auth.py b/packages/pandas-gbq/pandas_gbq/auth.py index 9f32bac77e03..b6dca129b4c8 100644 --- a/packages/pandas-gbq/pandas_gbq/auth.py +++ b/packages/pandas-gbq/pandas_gbq/auth.py @@ -12,40 +12,48 @@ logger = logging.getLogger(__name__) +CREDENTIALS_CACHE_DIRNAME = "pandas_gbq" +CREDENTIALS_CACHE_FILENAME = "bigquery_credentials.dat" SCOPES = ["https://www.googleapis.com/auth/bigquery"] +# The following constants are used for end-user authentication. +# It identifies the application that is requesting permission to access the +# BigQuery API on behalf of a G Suite or Gmail user. +# +# In a web application, the client secret would be kept secret, but this is not +# possible for applications that are installed locally on an end-user's +# machine. +# +# See: https://cloud.google.com/docs/authentication/end-user for details. +CLIENT_ID = ( + "495642085510-k0tmvj2m941jhre2nbqka17vqpjfddtd.apps.googleusercontent.com" +) +CLIENT_SECRET = "kOc9wMptUtxkcIFbtZCcrEAc" + def get_credentials( - private_key=None, - project_id=None, - reauth=False, - auth_local_webserver=False, - try_credentials=None, + private_key=None, project_id=None, reauth=False, auth_local_webserver=False ): - if try_credentials is None: - try_credentials = _try_credentials + import pydata_google_auth if private_key: return get_service_account_credentials(private_key) - # Try to retrieve Application Default Credentials - credentials, default_project = get_application_default_credentials( - try_credentials, project_id=project_id - ) - - if credentials: - return credentials, default_project - - credentials = get_user_account_credentials( - try_credentials, - project_id=project_id, - reauth=reauth, + credentials, default_project_id = pydata_google_auth.default( + SCOPES, + client_id=CLIENT_ID, + client_secret=CLIENT_SECRET, + credentials_cache=get_credentials_cache(reauth), auth_local_webserver=auth_local_webserver, ) + + project_id = project_id or default_project_id return credentials, project_id def get_service_account_credentials(private_key): + """DEPRECATED: Load service account credentials from key data or key path.""" + import google.auth.transport.requests from google.oauth2.service_account import Credentials @@ -87,233 +95,14 @@ def get_service_account_credentials(private_key): ) -def get_application_default_credentials(try_credentials, project_id=None): - """ - This method tries to retrieve the "default application credentials". - This could be useful for running code on Google Cloud Platform. - - Parameters - ---------- - project_id (str, optional): Override the default project ID. - - Returns - ------- - - GoogleCredentials, - If the default application credentials can be retrieved - from the environment. The retrieved credentials should also - have access to the project (project_id) on BigQuery. - - OR None, - If default application credentials can not be retrieved - from the environment. Or, the retrieved credentials do not - have access to the project (project_id) on BigQuery. - """ - import google.auth - from google.auth.exceptions import DefaultCredentialsError - - try: - credentials, default_project = google.auth.default(scopes=SCOPES) - except (DefaultCredentialsError, IOError): - return None, None - - # Even though we now have credentials, check that the credentials can be - # used with BigQuery. For example, we could be running on a GCE instance - # that does not allow the BigQuery scopes. - billing_project = project_id or default_project - return try_credentials(billing_project, credentials), billing_project - - -def get_user_account_credentials( - try_credentials, - project_id=None, - reauth=False, - auth_local_webserver=False, - credentials_path=None, -): - """Gets user account credentials. +def get_credentials_cache(reauth,): + import pydata_google_auth.cache - This method authenticates using user credentials, either loading saved - credentials from a file or by going through the OAuth flow. - - Parameters - ---------- - None - - Returns - ------- - GoogleCredentials : credentials - Credentials for the user with BigQuery access. - """ - from google_auth_oauthlib.flow import InstalledAppFlow - from oauthlib.oauth2.rfc6749.errors import OAuth2Error - - # Use the default credentials location under ~/.config and the - # equivalent directory on windows if the user has not specified a - # credentials path. - if not credentials_path: - credentials_path = get_default_credentials_path() - - # Previously, pandas-gbq saved user account credentials in the - # current working directory. If the bigquery_credentials.dat file - # exists in the current working directory, move the credentials to - # the new default location. - if os.path.isfile("bigquery_credentials.dat"): - os.rename("bigquery_credentials.dat", credentials_path) - - credentials = None - if not reauth: - credentials = load_user_account_credentials( - try_credentials, - project_id=project_id, - credentials_path=credentials_path, + if reauth: + return pydata_google_auth.cache.WriteOnlyCredentialsCache( + dirname=CREDENTIALS_CACHE_DIRNAME, + filename=CREDENTIALS_CACHE_FILENAME, ) - - client_config = { - "installed": { - "client_id": ( - "495642085510-k0tmvj2m941jhre2nbqka17vqpjfddtd" - ".apps.googleusercontent.com" - ), - "client_secret": "kOc9wMptUtxkcIFbtZCcrEAc", - "redirect_uris": ["urn:ietf:wg:oauth:2.0:oob"], - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://accounts.google.com/o/oauth2/token", - } - } - - if credentials is None: - app_flow = InstalledAppFlow.from_client_config( - client_config, scopes=SCOPES - ) - - try: - if auth_local_webserver: - credentials = app_flow.run_local_server() - else: - credentials = app_flow.run_console() - except OAuth2Error as ex: - raise pandas_gbq.exceptions.AccessDenied( - "Unable to get valid credentials: {0}".format(ex) - ) - - save_user_account_credentials(credentials, credentials_path) - - return credentials - - -def load_user_account_credentials( - try_credentials, project_id=None, credentials_path=None -): - """ - Loads user account credentials from a local file. - - .. versionadded 0.2.0 - - Parameters - ---------- - None - - Returns - ------- - - GoogleCredentials, - If the credentials can loaded. The retrieved credentials should - also have access to the project (project_id) on BigQuery. - - OR None, - If credentials can not be loaded from a file. Or, the retrieved - credentials do not have access to the project (project_id) - on BigQuery. - """ - import google.auth.transport.requests - from google.oauth2.credentials import Credentials - - try: - with open(credentials_path) as credentials_file: - credentials_json = json.load(credentials_file) - except (IOError, ValueError): - return None - - credentials = Credentials( - token=credentials_json.get("access_token"), - refresh_token=credentials_json.get("refresh_token"), - id_token=credentials_json.get("id_token"), - token_uri=credentials_json.get("token_uri"), - client_id=credentials_json.get("client_id"), - client_secret=credentials_json.get("client_secret"), - scopes=credentials_json.get("scopes"), + return pydata_google_auth.cache.ReadWriteCredentialsCache( + dirname=CREDENTIALS_CACHE_DIRNAME, filename=CREDENTIALS_CACHE_FILENAME ) - - # Refresh the token before trying to use it. - request = google.auth.transport.requests.Request() - credentials.refresh(request) - - return try_credentials(project_id, credentials) - - -def get_default_credentials_path(): - """ - Gets the default path to the BigQuery credentials - - .. versionadded 0.3.0 - - Returns - ------- - Path to the BigQuery credentials - """ - if os.name == "nt": - config_path = os.environ["APPDATA"] - else: - config_path = os.path.join(os.path.expanduser("~"), ".config") - - config_path = os.path.join(config_path, "pandas_gbq") - - # Create a pandas_gbq directory in an application-specific hidden - # user folder on the operating system. - if not os.path.exists(config_path): - os.makedirs(config_path) - - return os.path.join(config_path, "bigquery_credentials.dat") - - -def save_user_account_credentials(credentials, credentials_path): - """ - Saves user account credentials to a local file. - - .. versionadded 0.2.0 - """ - try: - with open(credentials_path, "w") as credentials_file: - credentials_json = { - "refresh_token": credentials.refresh_token, - "id_token": credentials.id_token, - "token_uri": credentials.token_uri, - "client_id": credentials.client_id, - "client_secret": credentials.client_secret, - "scopes": credentials.scopes, - } - json.dump(credentials_json, credentials_file) - except IOError: - logger.warning("Unable to save credentials.") - - -def _try_credentials(project_id, credentials): - from google.cloud import bigquery - import google.api_core.exceptions - import google.auth.exceptions - - if not credentials: - return None - if not project_id: - return credentials - - try: - client = bigquery.Client(project=project_id, credentials=credentials) - # Check if the application has rights to the BigQuery project - client.query("SELECT 1").result() - return credentials - except google.api_core.exceptions.GoogleAPIError: - return None - except google.auth.exceptions.RefreshError: - # Sometimes (such as on Travis) google-auth returns GCE credentials, - # but fetching the token for those credentials doesn't actually work. - # See: - # https://github.com/googleapis/google-auth-library-python/issues/287 - return None diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index c71b6cc5d192..d35eba05cbf6 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -68,6 +68,13 @@ def _check_google_client_version(): def _test_google_api_imports(): + try: + import pydata_google_auth + except ImportError as ex: + raise ImportError( + "pandas-gbq requires pydata-google-auth: {0}".format(ex) + ) + try: from google_auth_oauthlib.flow import InstalledAppFlow # noqa except ImportError as ex: @@ -297,7 +304,6 @@ def __init__( auth_local_webserver=False, dialect="legacy", location=None, - try_credentials=None, credentials=None, ): global context @@ -313,7 +319,6 @@ def __init__( self.auth_local_webserver = auth_local_webserver self.dialect = dialect self.credentials = credentials - self.credentials_path = _get_credentials_file() default_project = None # Load credentials from cache. @@ -328,7 +333,6 @@ def __init__( project_id=project_id, reauth=reauth, auth_local_webserver=auth_local_webserver, - try_credentials=try_credentials, ) if self.project_id is None: @@ -635,10 +639,6 @@ def delete_and_recreate_table(self, dataset_id, table_id, table_schema): table.create(table_id, table_schema) -def _get_credentials_file(): - return os.environ.get("PANDAS_GBQ_CREDENTIALS_FILE") - - def _parse_schema(schema_fields): # see: # http://pandas.pydata.org/pandas-docs/dev/missing_data.html @@ -1003,9 +1003,6 @@ def to_gbq( reauth=reauth, auth_local_webserver=auth_local_webserver, location=location, - # Avoid reads when writing tables. - # https://github.com/pydata/pandas-gbq/issues/202 - try_credentials=lambda project, creds: creds, credentials=credentials, private_key=private_key, ) diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index bd0c0d1184c8..e53d43f5d647 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -19,6 +19,7 @@ def readme(): INSTALL_REQUIRES = [ "setuptools", "pandas", + "pydata-google-auth", "google-auth", "google-auth-oauthlib", "google-cloud-bigquery>=0.32.0", diff --git a/packages/pandas-gbq/tests/system/test_auth.py b/packages/pandas-gbq/tests/system/test_auth.py index c48f45200abc..d02f153a365e 100644 --- a/packages/pandas-gbq/tests/system/test_auth.py +++ b/packages/pandas-gbq/tests/system/test_auth.py @@ -9,6 +9,35 @@ from pandas_gbq import auth +def mock_default_credentials(scopes=None, request=None): + return (None, None) + + +def _try_credentials(project_id, credentials): + from google.cloud import bigquery + import google.api_core.exceptions + import google.auth.exceptions + + if not credentials: + return None + if not project_id: + return credentials + + try: + client = bigquery.Client(project=project_id, credentials=credentials) + # Check if the application has rights to the BigQuery project + client.query("SELECT 1").result() + return credentials + except google.api_core.exceptions.GoogleAPIError: + return None + except google.auth.exceptions.RefreshError: + # Sometimes (such as on Travis) google-auth returns GCE credentials, + # but fetching the token for those credentials doesn't actually work. + # See: + # https://github.com/googleapis/google-auth-library-python/issues/287 + return None + + def _check_if_can_get_correct_default_credentials(): # Checks if "Application Default Credentials" can be fetched # from the environment the tests are running in. @@ -26,7 +55,7 @@ def _check_if_can_get_correct_default_credentials(): except (DefaultCredentialsError, IOError): return False - return auth._try_credentials(project, credentials) is not None + return _try_credentials(project, credentials) is not None def test_should_be_able_to_get_valid_credentials(project_id, private_key_path): @@ -43,7 +72,7 @@ def test_get_service_account_credentials_private_key_path(private_key_path): private_key_path ) assert isinstance(credentials, Credentials) - assert auth._try_credentials(project_id, credentials) is not None + assert _try_credentials(project_id, credentials) is not None def test_get_service_account_credentials_private_key_contents( @@ -55,71 +84,44 @@ def test_get_service_account_credentials_private_key_contents( private_key_contents ) assert isinstance(credentials, Credentials) - assert auth._try_credentials(project_id, credentials) is not None - - -def test_get_application_default_credentials_does_not_throw_error(): - if _check_if_can_get_correct_default_credentials(): - # Can get real credentials, so mock it out to fail. - from google.auth.exceptions import DefaultCredentialsError - - with mock.patch( - "google.auth.default", side_effect=DefaultCredentialsError() - ): - credentials, _ = auth.get_application_default_credentials( - try_credentials=auth._try_credentials - ) - else: - credentials, _ = auth.get_application_default_credentials( - try_credentials=auth._try_credentials - ) - assert credentials is None - - -def test_get_application_default_credentials_returns_credentials(): - if not _check_if_can_get_correct_default_credentials(): - pytest.skip("Cannot get default_credentials " "from the environment!") - from google.auth.credentials import Credentials - - credentials, default_project = auth.get_application_default_credentials( - try_credentials=auth._try_credentials - ) - - assert isinstance(credentials, Credentials) - assert default_project is not None + assert _try_credentials(project_id, credentials) is not None @pytest.mark.local_auth -def test_get_user_account_credentials_bad_file_returns_user_credentials(): +def test_get_credentials_bad_file_returns_user_credentials( + project_id, monkeypatch +): + import google.auth from google.auth.credentials import Credentials + monkeypatch.setattr(google.auth, "default", mock_default_credentials) + with mock.patch("__main__.open", side_effect=IOError()): - credentials = auth.get_user_account_credentials( - try_credentials=auth._try_credentials + credentials, _ = auth.get_credentials( + project_id=project_id, auth_local_webserver=True ) assert isinstance(credentials, Credentials) @pytest.mark.local_auth -def test_get_user_account_credentials_returns_credentials(project_id): - from google.auth.credentials import Credentials +def test_get_credentials_user_credentials_with_reauth(project_id, monkeypatch): + import google.auth + + monkeypatch.setattr(google.auth, "default", mock_default_credentials) - credentials = auth.get_user_account_credentials( - project_id=project_id, - auth_local_webserver=True, - try_credentials=auth._try_credentials, + credentials, _ = auth.get_credentials( + project_id=project_id, reauth=True, auth_local_webserver=True ) - assert isinstance(credentials, Credentials) + assert credentials.valid @pytest.mark.local_auth -def test_get_user_account_credentials_reauth_returns_credentials(project_id): - from google.auth.credentials import Credentials +def test_get_credentials_user_credentials(project_id, monkeypatch): + import google.auth + + monkeypatch.setattr(google.auth, "default", mock_default_credentials) - credentials = auth.get_user_account_credentials( - project_id=project_id, - auth_local_webserver=True, - reauth=True, - try_credentials=auth._try_credentials, + credentials, _ = auth.get_credentials( + project_id=project_id, auth_local_webserver=True ) - assert isinstance(credentials, Credentials) + assert credentials.valid diff --git a/packages/pandas-gbq/tests/unit/test_auth.py b/packages/pandas-gbq/tests/unit/test_auth.py index d8107a40b78b..f9f0dc940d91 100644 --- a/packages/pandas-gbq/tests/unit/test_auth.py +++ b/packages/pandas-gbq/tests/unit/test_auth.py @@ -86,6 +86,7 @@ def mock_default_credentials(scopes=None, request=None): def test_get_credentials_load_user_no_default(monkeypatch): import google.auth import google.auth.credentials + import pydata_google_auth.cache def mock_default_credentials(scopes=None, request=None): return (None, None) @@ -95,14 +96,12 @@ def mock_default_credentials(scopes=None, request=None): google.auth.credentials.Credentials ) - def mock_load_credentials( - try_credentials, project_id=None, credentials_path=None - ): - return mock_user_credentials - - monkeypatch.setattr( - auth, "load_user_account_credentials", mock_load_credentials + mock_cache = mock.create_autospec( + pydata_google_auth.cache.CredentialsCache ) + mock_cache.load.return_value = mock_user_credentials + + monkeypatch.setattr(auth, "get_credentials_cache", lambda _: mock_cache) credentials, project = auth.get_credentials() assert project is None diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index 004360288e94..c4d8fe2e8181 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -29,6 +29,15 @@ def mock_none_credentials(*args, **kwargs): return None, None +def mock_get_credentials_no_project(*args, **kwargs): + import google.auth.credentials + + mock_credentials = mock.create_autospec( + google.auth.credentials.Credentials + ) + return mock_credentials, None + + def mock_get_credentials(*args, **kwargs): import google.auth.credentials @@ -50,22 +59,9 @@ def mock_get_user_credentials(*args, **kwargs): @pytest.fixture(autouse=True) def no_auth(monkeypatch): from pandas_gbq import auth + import pydata_google_auth - monkeypatch.setattr( - auth, "get_application_default_credentials", mock_get_credentials - ) - monkeypatch.setattr( - auth, "get_user_account_credentials", mock_get_user_credentials - ) - monkeypatch.setattr( - auth, "_try_credentials", lambda project_id, credentials: credentials - ) - - -def test_should_return_credentials_path_set_by_env_var(): - env = {"PANDAS_GBQ_CREDENTIALS_FILE": "/tmp/dummy.dat"} - with mock.patch.dict("os.environ", env): - assert gbq._get_credentials_file() == "/tmp/dummy.dat" + monkeypatch.setattr(pydata_google_auth, "default", mock_get_credentials) @pytest.mark.parametrize( @@ -97,10 +93,10 @@ def test_to_gbq_should_fail_if_invalid_table_name_passed(): def test_to_gbq_with_no_project_id_given_should_fail(monkeypatch): - from pandas_gbq import auth + import pydata_google_auth monkeypatch.setattr( - auth, "get_application_default_credentials", mock_none_credentials + pydata_google_auth, "default", mock_get_credentials_no_project ) with pytest.raises(ValueError) as exception: @@ -252,10 +248,10 @@ def test_to_gbq_doesnt_run_query( def test_read_gbq_with_no_project_id_given_should_fail(monkeypatch): - from pandas_gbq import auth + import pydata_google_auth monkeypatch.setattr( - auth, "get_application_default_credentials", mock_none_credentials + pydata_google_auth, "default", mock_get_credentials_no_project ) with pytest.raises(ValueError) as exception: From 6f409abe8dced239d89c46f475d04ccd7419a182 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 11 Jan 2019 09:33:14 -0800 Subject: [PATCH 167/519] Release 0.9.0 (#243) --- packages/pandas-gbq/docs/source/changelog.rst | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 961f289c7f45..6f3aa5cd41a3 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -3,12 +3,10 @@ Changelog .. _changelog-0.9.0: -0.9.0 / TBD ------------ - -Internal changes -~~~~~~~~~~~~~~~~ +0.9.0 / 2019-01-11 +------------------ +- Warn when deprecated ``private_key`` parameter is used (:issue:`240`) - **New dependency** Use the ``pydata-google-auth`` package for authentication. (:issue:`241`) From 56a34c283e5b3211824c9c026ce4e390ce5ff968 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 25 Jan 2019 12:14:38 -0800 Subject: [PATCH 168/519] ENH: Use standard SQL as default. (#245) Removes the warning when a SQL dialect is unspecified. --- packages/pandas-gbq/pandas_gbq/gbq.py | 15 ++++----------- packages/pandas-gbq/tests/unit/test_gbq.py | 5 ----- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index d35eba05cbf6..825482822532 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -302,7 +302,7 @@ def __init__( reauth=False, private_key=None, auth_local_webserver=False, - dialect="legacy", + dialect="standard", location=None, credentials=None, ): @@ -732,8 +732,8 @@ def read_gbq( http://google-auth-oauthlib.readthedocs.io/en/latest/reference/google_auth_oauthlib.flow.html#google_auth_oauthlib.flow.InstalledAppFlow.run_console .. versionadded:: 0.2.0 - dialect : str, default 'legacy' - Note: The default value is changing to 'standard' in a future verion. + dialect : str, default 'standard' + Note: The default value changed to 'standard' in version 0.10.0. SQL syntax dialect to use. Value can be one of: @@ -796,14 +796,7 @@ def read_gbq( dialect = context.dialect if dialect is None: - dialect = "legacy" - warnings.warn( - 'The default value for dialect is changing to "standard" in a ' - 'future version. Pass in dialect="legacy" or set ' - 'pandas_gbq.context.dialect="legacy" to disable this warning.', - FutureWarning, - stacklevel=2, - ) + dialect = "standard" _test_google_api_imports() diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index c4d8fe2e8181..ab16ec0b6385 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -429,11 +429,6 @@ def test_read_gbq_with_invalid_dialect(): assert "is not valid for dialect" in str(excinfo.value) -def test_read_gbq_without_dialect_warns_future_change(): - with pytest.warns(FutureWarning): - gbq.read_gbq("SELECT 1") - - def test_generate_bq_schema_deprecated(): # 11121 Deprecation of generate_bq_schema with pytest.warns(FutureWarning): From cabd052bc544b5845071881a9eb5689920a5e037 Mon Sep 17 00:00:00 2001 From: Brad Date: Fri, 25 Jan 2019 18:06:10 -0500 Subject: [PATCH 169/519] CLN: Close CSV BytesIO buffer after upload. (#244) (#246) While the buffer should eventually be destroyed/closed by the gc (the timing of this is implementation-dependent), it is safer practice to explicitly close immediately after upload. --- packages/pandas-gbq/pandas_gbq/load.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/load.py b/packages/pandas-gbq/pandas_gbq/load.py index 015f479d16e5..cb190f865351 100644 --- a/packages/pandas-gbq/pandas_gbq/load.py +++ b/packages/pandas-gbq/pandas_gbq/load.py @@ -79,10 +79,13 @@ def load_chunks( chunks = encode_chunks(dataframe, chunksize=chunksize) for remaining_rows, chunk_buffer in chunks: - yield remaining_rows - client.load_table_from_file( - chunk_buffer, - destination_table, - job_config=job_config, - location=location, - ).result() + try: + yield remaining_rows + client.load_table_from_file( + chunk_buffer, + destination_table, + job_config=job_config, + location=location, + ).result() + finally: + chunk_buffer.close() From c086d0939ae192704196da7a588c4d9056e95103 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 15 Feb 2019 15:37:12 -0800 Subject: [PATCH 170/519] TST: Use nox docker image for PIP CircleCI tests (#251) * TST: Use nox docker image for PIP CircleCI tests Fixes the CircleCI build. Clean up the noxfile to use the same requirements files as CI. Temporarily disables installing pandas nightlies and google-cloud-bigquery from master since they were taking several minutes to install in my local test environment. --- packages/pandas-gbq/.circleci/config.yml | 71 ++++++++--------- packages/pandas-gbq/.flake8 | 8 ++ packages/pandas-gbq/.gitignore | 2 + .../pandas-gbq/ci/requirements-2.7-0.19.2.pip | 1 - packages/pandas-gbq/ci/requirements-2.7.pip | 6 ++ .../pandas-gbq/ci/requirements-3.5-0.18.1.pip | 3 - packages/pandas-gbq/ci/requirements-3.5.pip | 5 ++ .../pandas-gbq/ci/requirements-3.6-MASTER.pip | 2 + packages/pandas-gbq/ci/requirements-3.6.pip | 3 + .../pandas-gbq/ci/requirements-3.7-0.23.4.pip | 0 packages/pandas-gbq/ci/requirements-3.7.pip | 3 + packages/pandas-gbq/noxfile.py | 77 ++++++++----------- packages/pandas-gbq/pandas_gbq/gbq.py | 3 +- .../pandas-gbq/tests/unit/test_context.py | 4 +- packages/pandas-gbq/tests/unit/test_gbq.py | 1 - 15 files changed, 97 insertions(+), 92 deletions(-) create mode 100644 packages/pandas-gbq/.flake8 delete mode 100644 packages/pandas-gbq/ci/requirements-2.7-0.19.2.pip create mode 100644 packages/pandas-gbq/ci/requirements-2.7.pip delete mode 100644 packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip create mode 100644 packages/pandas-gbq/ci/requirements-3.5.pip create mode 100644 packages/pandas-gbq/ci/requirements-3.6.pip delete mode 100644 packages/pandas-gbq/ci/requirements-3.7-0.23.4.pip create mode 100644 packages/pandas-gbq/ci/requirements-3.7.pip diff --git a/packages/pandas-gbq/.circleci/config.yml b/packages/pandas-gbq/.circleci/config.yml index c0132335dc38..4a88a1827b11 100644 --- a/packages/pandas-gbq/.circleci/config.yml +++ b/packages/pandas-gbq/.circleci/config.yml @@ -1,48 +1,48 @@ version: 2 jobs: # Pip - "pip-2.7-0.19.2": + "pip-2.7": docker: - - image: python:2.7 - environment: - PYTHON: "2.7" - PANDAS: "0.19.2" + - image: thekevjames/nox steps: - checkout - run: ci/config_auth.sh - - run: ci/run_pip.sh - "pip-3.5-0.18.1": + - run: nox -s unit-2.7 system-2.7 + + "pip-3.5": docker: - - image: python:3.5 - environment: - PYTHON: "3.5" - PANDAS: "0.18.1" + - image: thekevjames/nox steps: - checkout - run: ci/config_auth.sh - - run: ci/run_pip.sh - "pip-3.6-MASTER": + - run: nox -s unit-3.5 system-3.5 + + "pip-3.6": docker: - - image: python:3.6 - environment: - PYTHON: "3.6" - PANDAS: "MASTER" + - image: thekevjames/nox steps: - checkout - run: ci/config_auth.sh - - run: ci/run_pip.sh - # Coverage - - run: codecov - "pip-3.7-0.23.4": + - run: nox -s unit-3.6 system-3.6 + + "pip-3.7": docker: - - image: python:3.7 - environment: - PYTHON: "3.7" - PANDAS: "0.23.4" + - image: thekevjames/nox steps: - checkout - run: ci/config_auth.sh - - run: ci/run_pip.sh + - run: nox -s unit-3.7 system-3.7 cover + + "lint": + docker: + - image: thekevjames/nox + environment: + # Resolve "Python 3 was configured to use ASCII as encoding for the environment" + LC_ALL: C.UTF-8 + LANG: C.UTF-8 + steps: + - checkout + - run: nox -s lint # Conda "conda-3.6-0.20.1": @@ -56,20 +56,13 @@ jobs: - run: ci/config_auth.sh - run: ci/run_conda.sh - lint: - docker: - - image: python:3.6 - steps: - - checkout - - run: pip install nox - - run: nox -s lint workflows: version: 2 build: jobs: - - "pip-2.7-0.19.2" - - "pip-3.5-0.18.1" - - "pip-3.6-MASTER" - - "pip-3.7-0.23.4" - - "conda-3.6-0.20.1" - - lint \ No newline at end of file + - "pip-2.7" + - "pip-3.5" + - "pip-3.6" + - "pip-3.7" + - lint + - "conda-3.6-0.20.1" \ No newline at end of file diff --git a/packages/pandas-gbq/.flake8 b/packages/pandas-gbq/.flake8 new file mode 100644 index 000000000000..0574e0a3ab66 --- /dev/null +++ b/packages/pandas-gbq/.flake8 @@ -0,0 +1,8 @@ +[flake8] +ignore = E203, E266, E501, W503 +exclude = + # Standard linting exemptions. + __pycache__, + .git, + *.pyc, + conf.py diff --git a/packages/pandas-gbq/.gitignore b/packages/pandas-gbq/.gitignore index d1605d550f42..5427c785f17f 100644 --- a/packages/pandas-gbq/.gitignore +++ b/packages/pandas-gbq/.gitignore @@ -68,6 +68,8 @@ dist # wheel files *.whl **/wheelhouse/* +pip-wheel-metadata + # coverage .coverage .testmondata diff --git a/packages/pandas-gbq/ci/requirements-2.7-0.19.2.pip b/packages/pandas-gbq/ci/requirements-2.7-0.19.2.pip deleted file mode 100644 index 932a8957f78a..000000000000 --- a/packages/pandas-gbq/ci/requirements-2.7-0.19.2.pip +++ /dev/null @@ -1 +0,0 @@ -mock diff --git a/packages/pandas-gbq/ci/requirements-2.7.pip b/packages/pandas-gbq/ci/requirements-2.7.pip new file mode 100644 index 000000000000..48ceb439338e --- /dev/null +++ b/packages/pandas-gbq/ci/requirements-2.7.pip @@ -0,0 +1,6 @@ +mock +pandas==0.17.1 +google-auth==1.4.1 +google-auth-oauthlib==0.0.1 +google-cloud-bigquery==0.32.0 +pydata-google-auth==0.1.2 diff --git a/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip b/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip deleted file mode 100644 index 618d0ac9144a..000000000000 --- a/packages/pandas-gbq/ci/requirements-3.5-0.18.1.pip +++ /dev/null @@ -1,3 +0,0 @@ -google-auth==1.4.1 -google-auth-oauthlib==0.0.1 -google-cloud-bigquery==0.32.0 \ No newline at end of file diff --git a/packages/pandas-gbq/ci/requirements-3.5.pip b/packages/pandas-gbq/ci/requirements-3.5.pip new file mode 100644 index 000000000000..980d07001a73 --- /dev/null +++ b/packages/pandas-gbq/ci/requirements-3.5.pip @@ -0,0 +1,5 @@ +pandas==0.19.0 +google-auth==1.4.1 +google-auth-oauthlib==0.0.1 +google-cloud-bigquery==0.32.0 +pydata-google-auth==0.1.2 \ No newline at end of file diff --git a/packages/pandas-gbq/ci/requirements-3.6-MASTER.pip b/packages/pandas-gbq/ci/requirements-3.6-MASTER.pip index 9ca0b606e608..7f44634287b7 100644 --- a/packages/pandas-gbq/ci/requirements-3.6-MASTER.pip +++ b/packages/pandas-gbq/ci/requirements-3.6-MASTER.pip @@ -1,3 +1,5 @@ +--pre -f https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com/ pandas git+https://github.com/googleapis/google-cloud-python.git#egg=version_subpkg&subdirectory=api_core git+https://github.com/googleapis/google-cloud-python.git#egg=version_subpkg&subdirectory=core git+https://github.com/googleapis/google-cloud-python.git#egg=version_subpkg&subdirectory=bigquery +pydata-google-auth==0.1.2 \ No newline at end of file diff --git a/packages/pandas-gbq/ci/requirements-3.6.pip b/packages/pandas-gbq/ci/requirements-3.6.pip new file mode 100644 index 000000000000..25b7b34ec28f --- /dev/null +++ b/packages/pandas-gbq/ci/requirements-3.6.pip @@ -0,0 +1,3 @@ +pandas +pydata-google-auth +google-cloud-bigquery \ No newline at end of file diff --git a/packages/pandas-gbq/ci/requirements-3.7-0.23.4.pip b/packages/pandas-gbq/ci/requirements-3.7-0.23.4.pip deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/packages/pandas-gbq/ci/requirements-3.7.pip b/packages/pandas-gbq/ci/requirements-3.7.pip new file mode 100644 index 000000000000..710cb8b8a01e --- /dev/null +++ b/packages/pandas-gbq/ci/requirements-3.7.pip @@ -0,0 +1,3 @@ +pandas==0.24.0 +google-cloud-bigquery==1.9.0 +pydata-google-auth==0.1.2 \ No newline at end of file diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 104a34c4a60c..819742c39f9a 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -8,45 +8,34 @@ import nox -PANDAS_PRE_WHEELS = ( - "https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83" - ".ssl.cf2.rackcdn.com" -) - +supported_pythons = ["2.7", "3.5", "3.6", "3.7"] latest_python = "3.6" @nox.session -def test(session): - session.install("mock", "pytest", "pytest-cov") +def lint(session, python=latest_python): + session.install("black", "flake8") session.install("-e", ".") + session.run("flake8", "pandas_gbq") + session.run("flake8", "tests") + session.run("black", "--check", ".") - # Skip local auth tests on Travis. - additional_args = list(session.posargs) - if "TRAVIS_BUILD_DIR" in os.environ: - additional_args = additional_args + ["-m", "not local_auth"] - session.run( - "pytest", - os.path.join(".", "tests"), - "--quiet", - "--cov=pandas_gbq", - "--cov=tests.unit", - "--cov-report", - "xml:/tmp/pytest-cov.xml", - *additional_args - ) +@nox.session(python=latest_python) +def blacken(session): + session.install("black") + session.run("black", ".") -@nox.session +@nox.session(python=supported_pythons) def unit(session): session.install("mock", "pytest", "pytest-cov") session.install("-e", ".") session.run( "pytest", os.path.join(".", "tests", "unit"), - "--quiet", + "-v", "--cov=pandas_gbq", "--cov=tests.unit", "--cov-report", @@ -56,32 +45,32 @@ def unit(session): @nox.session -def test_earliest_deps(session, python="3.5"): - session.install( - "-r", os.path.join(".", "ci", "requirements-3.5-0.18.1.pip") - ) - test(session) +def cover(session, python=latest_python): + session.install("coverage", "pytest-cov") + session.run("coverage", "report", "--show-missing", "--fail-under=40") + session.run("coverage", "erase") -@nox.session -def test_latest_deps(session, python=latest_python): +@nox.session(python=supported_pythons) +def system(session): + session.install("pytest", "pytest-cov") session.install( - "--pre", "--upgrade", "--timeout=60", "-f", PANDAS_PRE_WHEELS, "pandas" + "-r", + os.path.join(".", "ci", "requirements-{}.pip".format(session.python)), ) session.install( - "-r", os.path.join(".", "ci", "requirements-3.6-MASTER.pip") + "-e", + ".", + # Use dependencies from requirements file instead. + # This enables testing with specific versions of the dependencies. + "--no-dependencies", ) - test(session) - - -@nox.session -def lint(session, python=latest_python): - session.install("black") - session.run("black", "--check", ".") + # Skip local auth tests on CI. + additional_args = list(session.posargs) + if "CIRCLECI" in os.environ: + additional_args = additional_args + ["-m", "not local_auth"] -@nox.session -def cover(session, python=latest_python): - session.install("coverage", "pytest-cov") - session.run("coverage", "report", "--show-missing", "--fail-under=40") - session.run("coverage", "erase") + session.run( + "pytest", os.path.join(".", "tests", "system"), "-v", *additional_args + ) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 825482822532..948fd980042c 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -1,5 +1,4 @@ import logging -import os import time import warnings from collections import OrderedDict @@ -69,7 +68,7 @@ def _check_google_client_version(): def _test_google_api_imports(): try: - import pydata_google_auth + import pydata_google_auth # noqa except ImportError as ex: raise ImportError( "pandas-gbq requires pydata-google-auth: {0}".format(ex) diff --git a/packages/pandas-gbq/tests/unit/test_context.py b/packages/pandas-gbq/tests/unit/test_context.py index ffb6a4e7480e..59a91501c208 100644 --- a/packages/pandas-gbq/tests/unit/test_context.py +++ b/packages/pandas-gbq/tests/unit/test_context.py @@ -46,11 +46,11 @@ def test_read_gbq_should_use_dialect(mock_bigquery_client): pandas_gbq.read_gbq("SELECT 1") _, kwargs = mock_bigquery_client.query.call_args - assert kwargs["job_config"].use_legacy_sql == True + assert kwargs["job_config"].use_legacy_sql pandas_gbq.context.dialect = "standard" pandas_gbq.read_gbq("SELECT 1") _, kwargs = mock_bigquery_client.query.call_args - assert kwargs["job_config"].use_legacy_sql == False + assert not kwargs["job_config"].use_legacy_sql pandas_gbq.context.dialect = None # Reset the global state. diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index ab16ec0b6385..4f1d18ad0f9d 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -58,7 +58,6 @@ def mock_get_user_credentials(*args, **kwargs): @pytest.fixture(autouse=True) def no_auth(monkeypatch): - from pandas_gbq import auth import pydata_google_auth monkeypatch.setattr(pydata_google_auth, "default", mock_get_credentials) From 6095f51f0a6c02e89ba36b4f05974d633f20f974 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 22 Feb 2019 16:38:01 -0800 Subject: [PATCH 171/519] CLN: Use `to_dataframe` to download query results. (#247) * CLN: Use `to_dataframe` to download query results. This allows us to remove logic for parsing the schema and align with google-cloud-bigquery. * Bumps the minimum google-cloud-bigquery version, because we need to use the new dtypes argument. * Cast to correct dtype in empty dataframes. * Improve the conda CI build to truly use dependencies from conda, not pip. Adds pydata-google-auth to conda deps. --- packages/pandas-gbq/benchmark/README.md | 16 ++++ .../benchmark/read_gbq_large_results.py | 8 ++ .../benchmark/read_gbq_small_results.py | 7 ++ packages/pandas-gbq/ci/requirements-2.7.pip | 2 +- packages/pandas-gbq/ci/requirements-3.5.pip | 2 +- .../ci/requirements-3.6-0.20.1.conda | 5 +- packages/pandas-gbq/ci/run_conda.sh | 2 +- packages/pandas-gbq/docs/source/changelog.rst | 18 ++++ packages/pandas-gbq/pandas_gbq/gbq.py | 82 +++++++++++------- packages/pandas-gbq/setup.py | 2 +- packages/pandas-gbq/tests/system/test_gbq.py | 86 +++++++++++++------ packages/pandas-gbq/tests/unit/test_gbq.py | 50 ++++------- 12 files changed, 178 insertions(+), 102 deletions(-) create mode 100644 packages/pandas-gbq/benchmark/README.md create mode 100644 packages/pandas-gbq/benchmark/read_gbq_large_results.py create mode 100644 packages/pandas-gbq/benchmark/read_gbq_small_results.py diff --git a/packages/pandas-gbq/benchmark/README.md b/packages/pandas-gbq/benchmark/README.md new file mode 100644 index 000000000000..5ede71d709bf --- /dev/null +++ b/packages/pandas-gbq/benchmark/README.md @@ -0,0 +1,16 @@ +# pandas-gbq benchmarks + +This directory contains a few scripts which are useful for performance +testing the pandas-gbq library. Use cProfile to time the script and see +details about where time is spent. To avoid timing how long BigQuery takes to +execute a query, run the benchmark twice to ensure the results are cached. + +## `read_gbq` + +Read a small table (a few KB). + + python -m cProfile --sort=cumtime read_gbq_small_results.py + +Read a large-ish table (100+ MB). + + python -m cProfile --sort=cumtime read_gbq_large_results.py diff --git a/packages/pandas-gbq/benchmark/read_gbq_large_results.py b/packages/pandas-gbq/benchmark/read_gbq_large_results.py new file mode 100644 index 000000000000..98d9ff53abeb --- /dev/null +++ b/packages/pandas-gbq/benchmark/read_gbq_large_results.py @@ -0,0 +1,8 @@ +import pandas_gbq + +# Select 163 MB worth of data, to time how long it takes to download large +# result sets. +df = pandas_gbq.read_gbq( + "SELECT * FROM `bigquery-public-data.usa_names.usa_1910_2013`", + dialect="standard", +) diff --git a/packages/pandas-gbq/benchmark/read_gbq_small_results.py b/packages/pandas-gbq/benchmark/read_gbq_small_results.py new file mode 100644 index 000000000000..8e91b0a0cfd0 --- /dev/null +++ b/packages/pandas-gbq/benchmark/read_gbq_small_results.py @@ -0,0 +1,7 @@ +import pandas_gbq + +# Select a few KB worth of data, to time downloading small result sets. +df = pandas_gbq.read_gbq( + "SELECT * FROM `bigquery-public-data.utility_us.country_code_iso`", + dialect="standard", +) diff --git a/packages/pandas-gbq/ci/requirements-2.7.pip b/packages/pandas-gbq/ci/requirements-2.7.pip index 48ceb439338e..10300f125329 100644 --- a/packages/pandas-gbq/ci/requirements-2.7.pip +++ b/packages/pandas-gbq/ci/requirements-2.7.pip @@ -2,5 +2,5 @@ mock pandas==0.17.1 google-auth==1.4.1 google-auth-oauthlib==0.0.1 -google-cloud-bigquery==0.32.0 +google-cloud-bigquery==1.9.0 pydata-google-auth==0.1.2 diff --git a/packages/pandas-gbq/ci/requirements-3.5.pip b/packages/pandas-gbq/ci/requirements-3.5.pip index 980d07001a73..41a4189182c2 100644 --- a/packages/pandas-gbq/ci/requirements-3.5.pip +++ b/packages/pandas-gbq/ci/requirements-3.5.pip @@ -1,5 +1,5 @@ pandas==0.19.0 google-auth==1.4.1 google-auth-oauthlib==0.0.1 -google-cloud-bigquery==0.32.0 +google-cloud-bigquery==1.9.0 pydata-google-auth==0.1.2 \ No newline at end of file diff --git a/packages/pandas-gbq/ci/requirements-3.6-0.20.1.conda b/packages/pandas-gbq/ci/requirements-3.6-0.20.1.conda index a057399d3dbe..1c7eb3f22b84 100644 --- a/packages/pandas-gbq/ci/requirements-3.6-0.20.1.conda +++ b/packages/pandas-gbq/ci/requirements-3.6-0.20.1.conda @@ -1,6 +1,5 @@ -google-auth -google-auth-oauthlib -google-cloud-bigquery==0.32.0 +pydata-google-auth +google-cloud-bigquery==1.9.0 pytest pytest-cov codecov diff --git a/packages/pandas-gbq/ci/run_conda.sh b/packages/pandas-gbq/ci/run_conda.sh index 597693286a97..60ae6ff0f079 100755 --- a/packages/pandas-gbq/ci/run_conda.sh +++ b/packages/pandas-gbq/ci/run_conda.sh @@ -21,7 +21,7 @@ fi REQ="ci/requirements-${PYTHON}-${PANDAS}" conda install -q --file "$REQ.conda"; -python setup.py develop +python setup.py develop --no-deps # Run the tests $DIR/run_tests.sh diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 6f3aa5cd41a3..60861dea5538 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,6 +1,24 @@ Changelog ========= +.. _changelog-0.10.0: + +0.10.0 / TBD +------------ + +Dependency updates +~~~~~~~~~~~~~~~~~~ + +- Update the minimum version of ``google-cloud-bigquery`` to 1.9.0. + (:issue:`247`) + +Internal changes +~~~~~~~~~~~~~~~~ + +- Use ``to_dataframe()`` from ``google-cloud-bigquery`` in the ``read_gbq()`` + function. (:issue:`247`) + + .. _changelog-0.9.0: 0.9.0 / 2019-01-11 diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 948fd980042c..13d65669dd9c 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -1,11 +1,9 @@ import logging import time import warnings -from collections import OrderedDict from datetime import datetime import numpy as np -from pandas import DataFrame from pandas_gbq.exceptions import AccessDenied @@ -37,7 +35,7 @@ def _check_google_client_version(): raise ImportError("Could not import pkg_resources (setuptools).") # https://github.com/GoogleCloudPlatform/google-cloud-python/blob/master/bigquery/CHANGELOG.md - bigquery_minimum_version = pkg_resources.parse_version("0.32.0") + bigquery_minimum_version = pkg_resources.parse_version("1.9.0") BIGQUERY_INSTALLED_VERSION = pkg_resources.get_distribution( "google-cloud-bigquery" ).parsed_version @@ -482,15 +480,16 @@ def run_query(self, query, **kwargs): rows_iter = query_reply.result() except self.http_error as ex: self.process_http_error(ex) - result_rows = list(rows_iter) - total_rows = rows_iter.total_rows - schema = { - "fields": [field.to_api_repr() for field in rows_iter.schema] - } - logger.debug("Got {} rows.\n".format(total_rows)) + schema_fields = [field.to_api_repr() for field in rows_iter.schema] + nullsafe_dtypes = _bqschema_to_nullsafe_dtypes(schema_fields) + df = rows_iter.to_dataframe(dtypes=nullsafe_dtypes) + + if df.empty: + df = _cast_empty_df_dtypes(schema_fields, df) - return schema, result_rows + logger.debug("Got {} rows.\n".format(rows_iter.total_rows)) + return df def load_data( self, @@ -638,45 +637,62 @@ def delete_and_recreate_table(self, dataset_id, table_id, table_schema): table.create(table_id, table_schema) -def _parse_schema(schema_fields): - # see: +def _bqschema_to_nullsafe_dtypes(schema_fields): + # Only specify dtype when the dtype allows nulls. Otherwise, use pandas's + # default dtype choice. + # + # See: # http://pandas.pydata.org/pandas-docs/dev/missing_data.html # #missing-data-casting-rules-and-indexing dtype_map = { "FLOAT": np.dtype(float), + # Even though TIMESTAMPs are timezone-aware in BigQuery, pandas doesn't + # support datetime64[ns, UTC] as dtype in DataFrame constructors. See: + # https://github.com/pandas-dev/pandas/issues/12513 "TIMESTAMP": "datetime64[ns]", "TIME": "datetime64[ns]", "DATE": "datetime64[ns]", "DATETIME": "datetime64[ns]", - "BOOLEAN": bool, - "INTEGER": np.int64, } + dtypes = {} for field in schema_fields: name = str(field["name"]) if field["mode"].upper() == "REPEATED": - yield name, object - else: - dtype = dtype_map.get(field["type"].upper()) - yield name, dtype + continue + + dtype = dtype_map.get(field["type"].upper()) + if dtype: + dtypes[name] = dtype + return dtypes -def _parse_data(schema, rows): - column_dtypes = OrderedDict(_parse_schema(schema["fields"])) - df = DataFrame(data=(iter(r) for r in rows), columns=column_dtypes.keys()) +def _cast_empty_df_dtypes(schema_fields, df): + """Cast any columns in an empty dataframe to correct type. - for column in df: - dtype = column_dtypes[column] - null_safe = ( - df[column].notnull().all() - or dtype == float - or dtype == "datetime64[ns]" + In an empty dataframe, pandas cannot choose a dtype unless one is + explicitly provided. The _bqschema_to_nullsafe_dtypes() function only + provides dtypes when the dtype safely handles null values. This means + that empty int64 and boolean columns are incorrectly classified as + ``object``. + """ + if not df.empty: + raise ValueError( + "DataFrame must be empty in order to cast non-nullsafe dtypes" ) - if dtype and null_safe: - df[column] = df[column].astype( - column_dtypes[column], errors="ignore" - ) + + dtype_map = {"BOOLEAN": bool, "INTEGER": np.int64} + + for field in schema_fields: + column = str(field["name"]) + if field["mode"].upper() == "REPEATED": + continue + + dtype = dtype_map.get(field["type"].upper()) + if dtype: + df[column] = df[column].astype(dtype) + return df @@ -825,8 +841,8 @@ def read_gbq( credentials=credentials, private_key=private_key, ) - schema, rows = connector.run_query(query, configuration=configuration) - final_df = _parse_data(schema, rows) + + final_df = connector.run_query(query, configuration=configuration) # Reindex the DataFrame on the provided column if index_col is not None: diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index e53d43f5d647..e5e40505e38c 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -22,7 +22,7 @@ def readme(): "pydata-google-auth", "google-auth", "google-auth-oauthlib", - "google-cloud-bigquery>=0.32.0", + "google-cloud-bigquery>=1.9.0", ] extras = {"tqdm": "tqdm>=4.23.0"} diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index dde34cb139e0..82753a382af1 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -6,11 +6,12 @@ import google.oauth2.service_account import numpy as np +import pandas import pandas.util.testing as tm -import pytest -import pytz from pandas import DataFrame, NaT, compat from pandas.compat import range, u +import pytest +import pytz from pandas_gbq import gbq @@ -138,14 +139,6 @@ def test_should_be_able_to_get_a_bigquery_client(self, gbq_connector): bigquery_client = gbq_connector.get_client() assert bigquery_client is not None - def test_should_be_able_to_get_schema_from_query(self, gbq_connector): - schema, pages = gbq_connector.run_query("SELECT 1") - assert schema is not None - - def test_should_be_able_to_get_results_from_query(self, gbq_connector): - schema, pages = gbq_connector.run_query("SELECT 1") - assert pages is not None - def test_should_read(project, credentials): query = 'SELECT "PI" AS valid_string' @@ -319,7 +312,8 @@ def test_should_properly_handle_timestamp_unix_epoch(self, project_id): tm.assert_frame_equal( df, DataFrame( - {"unix_epoch": [np.datetime64("1970-01-01T00:00:00.000000Z")]} + {"unix_epoch": ["1970-01-01T00:00:00.000000Z"]}, + dtype="datetime64[ns]", ), ) @@ -334,11 +328,38 @@ def test_should_properly_handle_arbitrary_timestamp(self, project_id): tm.assert_frame_equal( df, DataFrame( - { - "valid_timestamp": [ - np.datetime64("2004-09-15T05:00:00.000000Z") - ] - } + {"valid_timestamp": ["2004-09-15T05:00:00.000000Z"]}, + dtype="datetime64[ns]", + ), + ) + + def test_should_properly_handle_datetime_unix_epoch(self, project_id): + query = 'SELECT DATETIME("1970-01-01 00:00:00") AS unix_epoch' + df = gbq.read_gbq( + query, + project_id=project_id, + credentials=self.credentials, + dialect="legacy", + ) + tm.assert_frame_equal( + df, + DataFrame( + {"unix_epoch": ["1970-01-01T00:00:00"]}, dtype="datetime64[ns]" + ), + ) + + def test_should_properly_handle_arbitrary_datetime(self, project_id): + query = 'SELECT DATETIME("2004-09-15 05:00:00") AS valid_timestamp' + df = gbq.read_gbq( + query, + project_id=project_id, + credentials=self.credentials, + dialect="legacy", + ) + tm.assert_frame_equal( + df, + DataFrame( + {"valid_timestamp": [np.datetime64("2004-09-15T05:00:00")]} ), ) @@ -346,7 +367,7 @@ def test_should_properly_handle_arbitrary_timestamp(self, project_id): "expression, type_", [ ("current_date()", " Date: Tue, 26 Feb 2019 16:08:20 +0100 Subject: [PATCH 172/519] BUG: resolve divide by 0 error when uploading empty dataframe (#252) * resolve divide by 0 error when uploading empty dataframe * reformat with black * add unit test when uploading empty dataframe * add empty data upload system test * remove empty df unit test * update empty df * add 0.10.0 release note * update release note version number to 0.11.0 * update empty dataframe bug fix in change log --- packages/pandas-gbq/docs/source/changelog.rst | 4 +++- packages/pandas-gbq/pandas_gbq/gbq.py | 4 ++-- packages/pandas-gbq/tests/system/test_gbq.py | 22 +++++++++++++++++++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 60861dea5538..3b43ccd30f2e 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -6,6 +6,8 @@ Changelog 0.10.0 / TBD ------------ +- This fixes a bug where pandas-gbq could not upload an empty database. (:issue:`237`) + Dependency updates ~~~~~~~~~~~~~~~~~~ @@ -235,4 +237,4 @@ Initial release of transfered code from `pandas `__ -- :func:`read_gbq` now stores ``INTEGER`` columns as ``dtype=object`` if they contain ``NULL`` values. Otherwise they are stored as ``int64``. This prevents precision lost for integers greather than 2**53. Furthermore ``FLOAT`` columns with values above 10**4 are no longer casted to ``int64`` which also caused precision loss `pandas-GH#14064 `__, and `pandas-GH#14305 `__ +- :func:`read_gbq` now stores ``INTEGER`` columns as ``dtype=object`` if they contain ``NULL`` values. Otherwise they are stored as ``int64``. This prevents precision lost for integers greather than 2**53. Furthermore ``FLOAT`` columns with values above 10**4 are no longer casted to ``int64`` which also caused precision loss `pandas-GH#14064 `__, and `pandas-GH#14305 `__ \ No newline at end of file diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 13d65669dd9c..b59c3f94fd46 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -518,8 +518,8 @@ def load_data( chunks = tqdm.tqdm(chunks) for remaining_rows in chunks: logger.info( - "\rLoad is {0}% Complete".format( - ((total_rows - remaining_rows) * 100) / total_rows + "\r{} out of {} rows loaded.".format( + total_rows - remaining_rows, total_rows ) ) except self.http_error as ex: diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index 82753a382af1..2153926d2de3 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -924,6 +924,28 @@ def test_upload_data(self, project_id): ) assert result["num_rows"][0] == test_size + def test_upload_empty_data(self, project_id): + test_id = "data_with_0_rows" + test_size = 0 + df = DataFrame() + + gbq.to_gbq( + df, + self.destination_table + test_id, + project_id, + credentials=self.credentials, + ) + + result = gbq.read_gbq( + "SELECT COUNT(*) AS num_rows FROM {0}".format( + self.destination_table + test_id + ), + project_id=project_id, + credentials=self.credentials, + dialect="legacy", + ) + assert result["num_rows"][0] == test_size + def test_upload_data_if_table_exists_fail(self, project_id): test_id = "2" test_size = 10 From 7b3b847cbb31380aec45a6f2d551c638207207d0 Mon Sep 17 00:00:00 2001 From: John Paton Date: Tue, 12 Mar 2019 15:23:23 +0100 Subject: [PATCH 173/519] Update intro.rst (#256) --- packages/pandas-gbq/docs/source/intro.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/pandas-gbq/docs/source/intro.rst b/packages/pandas-gbq/docs/source/intro.rst index 49f46614b3c5..0e1a6a3b0374 100644 --- a/packages/pandas-gbq/docs/source/intro.rst +++ b/packages/pandas-gbq/docs/source/intro.rst @@ -20,7 +20,6 @@ more verbose logs, you can do something like: .. code-block:: ipython import logging - import sys logger = logging.getLogger('pandas_gbq') logger.setLevel(logging.DEBUG) - logger.addHandler(logging.StreamHandler(stream=sys.stdout)) + logger.addHandler(logging.StreamHandler()) From 6725fe51303c23405838c3d5b2cf14c152a5fc08 Mon Sep 17 00:00:00 2001 From: John Paton Date: Tue, 12 Mar 2019 19:17:15 +0100 Subject: [PATCH 174/519] ENH: Allow partial table schema in to_gbq() table_schema (#218) (#257) * ENH: Allow partial table schema in to_gbq * CLN: applied black * BUG: make update_schema python 2.7 compatible * DOC: update docs to allow for a subset of columns in to_gbq table_schema * DOC: what's new * DOC: close parens around issue in changelog --- packages/pandas-gbq/docs/source/changelog.rst | 7 ++- packages/pandas-gbq/pandas_gbq/gbq.py | 21 +++++++-- packages/pandas-gbq/pandas_gbq/schema.py | 29 ++++++++++++ packages/pandas-gbq/tests/unit/test_schema.py | 46 +++++++++++++++++++ 4 files changed, 97 insertions(+), 6 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 3b43ccd30f2e..e3c0edd7ad87 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -20,6 +20,11 @@ Internal changes - Use ``to_dataframe()`` from ``google-cloud-bigquery`` in the ``read_gbq()`` function. (:issue:`247`) +Enhancements +~~~~~~~~~~~~ +- Allow ``table_schema`` in :func:`to_gbq` to contain only a subset of columns, + with the rest being populated using the DataFrame dtypes (:issue:`218`) + (contributed by @johnpaton) .. _changelog-0.9.0: @@ -237,4 +242,4 @@ Initial release of transfered code from `pandas `__ -- :func:`read_gbq` now stores ``INTEGER`` columns as ``dtype=object`` if they contain ``NULL`` values. Otherwise they are stored as ``int64``. This prevents precision lost for integers greather than 2**53. Furthermore ``FLOAT`` columns with values above 10**4 are no longer casted to ``int64`` which also caused precision loss `pandas-GH#14064 `__, and `pandas-GH#14305 `__ \ No newline at end of file +- :func:`read_gbq` now stores ``INTEGER`` columns as ``dtype=object`` if they contain ``NULL`` values. Otherwise they are stored as ``int64``. This prevents precision lost for integers greather than 2**53. Furthermore ``FLOAT`` columns with values above 10**4 are no longer casted to ``int64`` which also caused precision loss `pandas-GH#14064 `__, and `pandas-GH#14305 `__ diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index b59c3f94fd46..2fa31e4f1038 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -939,9 +939,11 @@ def to_gbq( 'STRING'},...]``. If schema is not provided, it will be generated according to dtypes of DataFrame columns. - If schema is provided, it must contain all DataFrame columns. - pandas_gbq.gbq._generate_bq_schema() may be used to create an initial - schema, though it doesn't preserve column order. + If schema is provided, it may contain all or a subset of DataFrame + columns. If a subset is provided, the rest will be inferred from + the DataFrame dtypes. + pandas_gbq.gbq._generate_bq_schema() may be used to create an + initial schema, though it doesn't preserve column order. See BigQuery API documentation on available names of a field. .. versionadded:: 0.3.1 @@ -1023,10 +1025,13 @@ def to_gbq( credentials=connector.credentials, ) + default_schema = _generate_bq_schema(dataframe) if not table_schema: - table_schema = _generate_bq_schema(dataframe) + table_schema = default_schema else: - table_schema = dict(fields=table_schema) + table_schema = _update_bq_schema( + default_schema, dict(fields=table_schema) + ) # If table exists, check if_exists parameter if table.exists(table_id): @@ -1091,6 +1096,12 @@ def _generate_bq_schema(df, default_type="STRING"): return schema.generate_bq_schema(df, default_type=default_type) +def _update_bq_schema(schema_old, schema_new): + from pandas_gbq import schema + + return schema.update_schema(schema_old, schema_new) + + class _Table(GbqConnector): def __init__( self, diff --git a/packages/pandas-gbq/pandas_gbq/schema.py b/packages/pandas-gbq/pandas_gbq/schema.py index 3ca030252838..c59ed68ef345 100644 --- a/packages/pandas-gbq/pandas_gbq/schema.py +++ b/packages/pandas-gbq/pandas_gbq/schema.py @@ -31,3 +31,32 @@ def generate_bq_schema(dataframe, default_type="STRING"): ) return {"fields": fields} + + +def update_schema(schema_old, schema_new): + """ + Given an old BigQuery schema, update it with a new one. + + Where a field name is the same, the new will replace the old. Any + new fields not present in the old schema will be added. + + Arguments: + schema_old: the old schema to update + schema_new: the new schema which will overwrite/extend the old + """ + old_fields = schema_old["fields"] + new_fields = schema_new["fields"] + output_fields = list(old_fields) + + field_indices = {field["name"]: i for i, field in enumerate(output_fields)} + + for field in new_fields: + name = field["name"] + if name in field_indices: + # replace old field with new field of same name + output_fields[field_indices[name]] = field + else: + # add new field + output_fields.append(field) + + return {"fields": output_fields} diff --git a/packages/pandas-gbq/tests/unit/test_schema.py b/packages/pandas-gbq/tests/unit/test_schema.py index 74f22f29dec0..af3b204360ab 100644 --- a/packages/pandas-gbq/tests/unit/test_schema.py +++ b/packages/pandas-gbq/tests/unit/test_schema.py @@ -54,3 +54,49 @@ def test_generate_bq_schema(dataframe, expected_schema): schema = pandas_gbq.schema.generate_bq_schema(dataframe) assert schema == expected_schema + + +@pytest.mark.parametrize( + "schema_old,schema_new,expected_output", + [ + ( + {"fields": [{"name": "col1", "type": "INTEGER"}]}, + {"fields": [{"name": "col2", "type": "TIMESTAMP"}]}, + { + "fields": [ + {"name": "col1", "type": "INTEGER"}, + {"name": "col2", "type": "TIMESTAMP"}, + ] + }, + ), + ( + {"fields": [{"name": "col1", "type": "INTEGER"}]}, + {"fields": [{"name": "col1", "type": "BOOLEAN"}]}, + {"fields": [{"name": "col1", "type": "BOOLEAN"}]}, + ), + ( + { + "fields": [ + {"name": "col1", "type": "INTEGER"}, + {"name": "col2", "type": "INTEGER"}, + ] + }, + { + "fields": [ + {"name": "col2", "type": "BOOLEAN"}, + {"name": "col3", "type": "FLOAT"}, + ] + }, + { + "fields": [ + {"name": "col1", "type": "INTEGER"}, + {"name": "col2", "type": "BOOLEAN"}, + {"name": "col3", "type": "FLOAT"}, + ] + }, + ), + ], +) +def test_update_schema(schema_old, schema_new, expected_output): + output = pandas_gbq.schema.update_schema(schema_old, schema_new) + assert output == expected_output From 768b71888e30aeb41eb5c6c248097a8c317fba99 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 15 Mar 2019 13:38:56 -0700 Subject: [PATCH 175/519] DOC: Update "writing" docs with information about schema inference. (#259) * DOC: Update "writing" docs with information about schema inference. This commit started as a clean-up change to remove the unnecessary pandas_gbq.gbq._update_bq_schema method, but I then also updated the docs for to_gbq to be more clear about how the table_schema argument is to be used. I added a section to the writing.rst how-to guide about the table_schema parameter as well. Some of the "notes" in writing.rst were better as their own subsections. I moved the note on not to use BigQuery as a transactional database to the landing page. I link to the BigQuery sandox docs in the warning about creating a BigQuery account because you can follow those instructions to use BigQuery without entering credit card information. * Blacken --- packages/pandas-gbq/docs/source/conf.py | 11 +++- packages/pandas-gbq/docs/source/index.rst | 23 ++++--- packages/pandas-gbq/docs/source/tables.rst | 16 ----- packages/pandas-gbq/docs/source/writing.rst | 70 ++++++++++++++------- packages/pandas-gbq/noxfile.py | 23 +++++++ packages/pandas-gbq/pandas_gbq/gbq.py | 36 ++++++----- packages/pandas-gbq/pandas_gbq/schema.py | 2 + 7 files changed, 115 insertions(+), 66 deletions(-) delete mode 100644 packages/pandas-gbq/docs/source/tables.rst diff --git a/packages/pandas-gbq/docs/source/conf.py b/packages/pandas-gbq/docs/source/conf.py index 1959fc36b6f8..fad0ca018198 100644 --- a/packages/pandas-gbq/docs/source/conf.py +++ b/packages/pandas-gbq/docs/source/conf.py @@ -16,9 +16,12 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # +import datetime import os import sys +import pandas_gbq + # sys.path.insert(0, os.path.abspath('.')) # -- General configuration ------------------------------------------------ @@ -62,7 +65,9 @@ # General information about the project. project = u"pandas-gbq" -copyright = u"2017, PyData Development Team" +copyright = u"2017-{}, PyData Development Team".format( + datetime.datetime.now().year +) author = u"PyData Development Team" # The version info for the project you're documenting, acts as replacement for @@ -70,9 +75,9 @@ # built documents. # # The short X.Y version. -version = u"0.1.0" +version = pandas_gbq.__version__ # The full version, including alpha/beta/rc tags. -release = u"0.1.0" +release = version # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/packages/pandas-gbq/docs/source/index.rst b/packages/pandas-gbq/docs/source/index.rst index 8e895145a7b4..cbbdabf7289f 100644 --- a/packages/pandas-gbq/docs/source/index.rst +++ b/packages/pandas-gbq/docs/source/index.rst @@ -8,16 +8,24 @@ Welcome to pandas-gbq's documentation! The :mod:`pandas_gbq` module provides a wrapper for Google's BigQuery analytics web service to simplify retrieving results from BigQuery tables -using SQL-like queries. Result sets are parsed into a pandas -DataFrame with a shape and data types derived from the source table. -Additionally, DataFrames can be inserted into new BigQuery tables or appended -to existing tables. +using SQL-like queries. Result sets are parsed into a :class:`pandas.DataFrame` +with a shape and data types derived from the source table. Additionally, +DataFrames can be inserted into new BigQuery tables or appended to existing +tables. .. warning:: - To use this module, you will need a valid BigQuery account. Refer to the - `BigQuery Documentation `__ - for details on the service itself. + To use this module, you will need a valid BigQuery account. Use the + `BigQuery sandbox `__ to + try the service for free. + +While BigQuery uses standard SQL syntax, it has some important differences +from traditional databases both in functionality, API limitations (size and +quantity of queries or uploads), and how Google charges for use of the +service. BiqQuery is best for analyzing large sets of data quickly. It is not +a direct replacement for a transactional database. Refer to the `BigQuery +Documentation `__ for +details on the service itself. Contents: @@ -29,7 +37,6 @@ Contents: howto/authentication.rst reading.rst writing.rst - tables.rst api.rst contributing.rst changelog.rst diff --git a/packages/pandas-gbq/docs/source/tables.rst b/packages/pandas-gbq/docs/source/tables.rst deleted file mode 100644 index dcf891ee6b1b..000000000000 --- a/packages/pandas-gbq/docs/source/tables.rst +++ /dev/null @@ -1,16 +0,0 @@ -.. _create_tables: - -Creating Tables -=============== - -.. code-block:: ipython - - In [10]: gbq.generate_bq_schema(df, default_type='STRING') - - Out[10]: {'fields': [{'name': 'my_bool1', 'type': 'BOOLEAN'}, - {'name': 'my_bool2', 'type': 'BOOLEAN'}, - {'name': 'my_dates', 'type': 'TIMESTAMP'}, - {'name': 'my_float64', 'type': 'FLOAT'}, - {'name': 'my_int64', 'type': 'INTEGER'}, - {'name': 'my_string', 'type': 'STRING'}]} - diff --git a/packages/pandas-gbq/docs/source/writing.rst b/packages/pandas-gbq/docs/source/writing.rst index e649d7fdd1de..0c5b41e0cc7e 100644 --- a/packages/pandas-gbq/docs/source/writing.rst +++ b/packages/pandas-gbq/docs/source/writing.rst @@ -3,8 +3,8 @@ Writing DataFrames ================== -Assume we want to write a DataFrame ``df`` into a BigQuery table using -:func:`~pandas_gbq.to_gbq`. +Assume we want to write a :class:`~pandas.DataFrame` named ``df`` into a +BigQuery table using :func:`~pandas_gbq.to_gbq`. .. ipython:: python @@ -21,40 +21,62 @@ Assume we want to write a DataFrame ``df`` into a BigQuery table using .. code-block:: python - to_gbq(df, 'my_dataset.my_table', projectid) + import pandas_gbq + pandas_gbq.to_gbq(df, 'my_dataset.my_table', project_id=projectid) -.. note:: +The destination table and destination dataset will automatically be created +if they do not already exist. - The destination table and destination dataset will automatically be created if they do not already exist. -The ``if_exists`` argument can be used to dictate whether to ``'fail'``, ``'replace'`` -or ``'append'`` if the destination table already exists. The default value is ``'fail'``. +Writing to an Existing Table +---------------------------- + +Use the ``if_exists`` argument to dictate whether to ``'fail'``, +``'replace'`` or ``'append'`` if the destination table already exists. The +default value is ``'fail'``. For example, assume that ``if_exists`` is set to ``'fail'``. The following snippet will raise a ``TableCreationError`` if the destination table already exists. .. code-block:: python - to_gbq(df, 'my_dataset.my_table', projectid, if_exists='fail') + import pandas_gbq + pandas_gbq.to_gbq( + df, 'my_dataset.my_table', project_id=projectid, if_exists='fail', + ) + +If the ``if_exists`` argument is set to ``'append'``, the destination +dataframe will be written to the table using the defined table schema and +column types. The dataframe must contain fields (matching name and type) +currently in the destination table. + + +.. _writing-schema: + +Inferring the Table Schema +-------------------------- -.. note:: +The :func:`~pandas_gbq.to_gbq` method infers the BigQuery table schema based +on the dtypes of the uploaded :class:`~pandas.DataFrame`. - If the ``if_exists`` argument is set to ``'append'``, the destination - dataframe will be written to the table using the defined table schema and - column types. The dataframe must contain fields (matching name and type) - currently in the destination table. +========================= ================== +dtype BigQuery Data Type +========================= ================== +i (integer) INTEGER +b (boolean) BOOLEAN +f (float) FLOAT +O (object) STRING +S (zero-terminated bytes) STRING +U (Unicode string) STRING +M (datetime) TIMESTAMP +========================= ================== -.. note:: +If the data type inference does not suit your needs, supply a BigQuery schema +as the ``table_schema`` parameter of :func:`~pandas_gbq.to_gbq`. - If an error occurs while streaming data to BigQuery, see - `Troubleshooting BigQuery Errors `__. -.. note:: +Troubleshooting Errors +---------------------- - While BigQuery uses SQL-like syntax, it has some important differences - from traditional databases both in functionality, API limitations (size - and quantity of queries or uploads), and how Google charges for use of the - service. You should refer to `Google BigQuery documentation - `__ often as the service is always - evolving. BiqQuery is best for analyzing large sets of data quickly, but - it is not a direct replacement for a transactional database. +If an error occurs while writing data to BigQuery, see +`Troubleshooting BigQuery Errors `__. diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 819742c39f9a..c4c8a3c00382 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -5,6 +5,7 @@ import os import os.path +import shutil import nox @@ -51,6 +52,28 @@ def cover(session, python=latest_python): session.run("coverage", "erase") +@nox.session(python=latest_python) +def docs(session): + """Build the docs.""" + + session.install("-r", os.path.join("docs", "requirements-docs.txt")) + session.install("-e", ".") + + shutil.rmtree(os.path.join("docs", "source", "_build"), ignore_errors=True) + session.run( + "sphinx-build", + "-W", # warnings as errors + "-T", # show full traceback on exception + "-N", # no colors + "-b", + "html", + "-d", + os.path.join("docs", "source", "_build", "doctrees", ""), + os.path.join("docs", "source", ""), + os.path.join("docs", "source", "_build", "html", ""), + ) + + @nox.session(python=supported_pythons) def system(session): session.install("pytest", "pytest-cov") diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 2fa31e4f1038..3967b2841424 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -937,14 +937,18 @@ def to_gbq( List of BigQuery table fields to which according DataFrame columns conform to, e.g. ``[{'name': 'col1', 'type': 'STRING'},...]``. - If schema is not provided, it will be - generated according to dtypes of DataFrame columns. - If schema is provided, it may contain all or a subset of DataFrame - columns. If a subset is provided, the rest will be inferred from - the DataFrame dtypes. - pandas_gbq.gbq._generate_bq_schema() may be used to create an - initial schema, though it doesn't preserve column order. - See BigQuery API documentation on available names of a field. + + - If ``table_schema`` is provided, it may contain all or a subset of + DataFrame columns. If a subset is provided, the rest will be + inferred from the DataFrame dtypes. + - If ``table_schema`` is **not** provided, it will be + generated according to dtypes of DataFrame columns. See + `Inferring the Table Schema + `__. + for a description of the schema inference. + + See `BigQuery API documentation on valid column names + __. .. versionadded:: 0.3.1 location : str, optional @@ -985,6 +989,7 @@ def to_gbq( """ _test_google_api_imports() + from pandas_gbq import schema if verbose is not None and SHOW_VERBOSE_DEPRECATION: warnings.warn( @@ -1029,7 +1034,7 @@ def to_gbq( if not table_schema: table_schema = default_schema else: - table_schema = _update_bq_schema( + table_schema = schema.update_schema( default_schema, dict(fields=table_schema) ) @@ -1091,15 +1096,16 @@ def generate_bq_schema(df, default_type="STRING"): def _generate_bq_schema(df, default_type="STRING"): - from pandas_gbq import schema + """DEPRECATED: Given a dataframe, generate a Google BigQuery schema. - return schema.generate_bq_schema(df, default_type=default_type) - - -def _update_bq_schema(schema_old, schema_new): + This is a private method, but was used in external code to work around + issues in the default schema generation. Now that individual columns can + be overridden: https://github.com/pydata/pandas-gbq/issues/218, this + method can be removed after there is time to migrate away from this + method. """ from pandas_gbq import schema - return schema.update_schema(schema_old, schema_new) + return schema.generate_bq_schema(df, default_type=default_type) class _Table(GbqConnector): diff --git a/packages/pandas-gbq/pandas_gbq/schema.py b/packages/pandas-gbq/pandas_gbq/schema.py index c59ed68ef345..91963b7cca0c 100644 --- a/packages/pandas-gbq/pandas_gbq/schema.py +++ b/packages/pandas-gbq/pandas_gbq/schema.py @@ -11,6 +11,8 @@ def generate_bq_schema(dataframe, default_type="STRING"): does not exist in the schema. """ + # If you update this mapping, also update the table at + # `docs/source/writing.rst`. type_mapping = { "i": "INTEGER", "b": "BOOLEAN", From 8fd25982f4e485cb3f8af4b85687395eab7d9ffc Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 15 Mar 2019 15:39:47 -0700 Subject: [PATCH 176/519] BUG: avoid load jobs for empty dataframes (#255) * BUG: avoid load jobs for empty dataframes No reason to run a load job if there is no data to load. This avoids a "Empty schema specified for the load job." error when the DataFrame also contains no columns. * Blacken * Remove unused test_size variable. --- packages/pandas-gbq/pandas_gbq/gbq.py | 5 +++ packages/pandas-gbq/tests/system/test_gbq.py | 36 +++++++++++++++----- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 3967b2841424..d0c85a32bf7e 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -1063,6 +1063,11 @@ def to_gbq( else: table.create(table_id, table_schema) + if dataframe.empty: + # Create the table (if needed), but don't try to run a load job with an + # empty file. See: https://github.com/pydata/pandas-gbq/issues/237 + return + connector.load_data( dataframe, dataset_id, diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index 2153926d2de3..a98b679436ab 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -892,6 +892,8 @@ def test_tokyo(self, tokyo_dataset, tokyo_table, private_key_path): class TestToGBQIntegration(object): @pytest.fixture(autouse=True, scope="function") def setup(self, project, credentials, random_dataset_id): + from google.cloud import bigquery + # - PER-TEST FIXTURES - # put here any instruction you want to be run *BEFORE* *EVERY* test is # executed. @@ -900,6 +902,9 @@ def setup(self, project, credentials, random_dataset_id): ) self.destination_table = "{}.{}".format(random_dataset_id, TABLE_ID) self.credentials = credentials + self.bqclient = bigquery.Client( + project=project, credentials=credentials + ) def test_upload_data(self, project_id): test_id = "1" @@ -926,7 +931,6 @@ def test_upload_data(self, project_id): def test_upload_empty_data(self, project_id): test_id = "data_with_0_rows" - test_size = 0 df = DataFrame() gbq.to_gbq( @@ -936,15 +940,31 @@ def test_upload_empty_data(self, project_id): credentials=self.credentials, ) - result = gbq.read_gbq( - "SELECT COUNT(*) AS num_rows FROM {0}".format( - self.destination_table + test_id - ), - project_id=project_id, + table = self.bqclient.get_table(self.destination_table + test_id) + assert table.num_rows == 0 + assert len(table.schema) == 0 + + def test_upload_empty_data_with_schema(self, project_id): + test_id = "data_with_0_rows" + df = DataFrame( + { + "a": pandas.Series(dtype="int64"), + "b": pandas.Series(dtype="object"), + } + ) + + gbq.to_gbq( + df, + self.destination_table + test_id, + project_id, credentials=self.credentials, - dialect="legacy", ) - assert result["num_rows"][0] == test_size + + table = self.bqclient.get_table(self.destination_table + test_id) + assert table.num_rows == 0 + schema = table.schema + assert schema[0].field_type == "INTEGER" + assert schema[1].field_type == "STRING" def test_upload_data_if_table_exists_fail(self, project_id): test_id = "2" From aa6b84a6bfabaa4c466b93c0c030524056582fa2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9raud=20Le=20Falher?= Date: Fri, 22 Mar 2019 23:33:46 +0100 Subject: [PATCH 177/519] read project_id from credentials if available (#258) * read project_id from credentials if available * Move project inference from credentials to shared GbqConnector To more easily test the project ID inference, this commit also sends the project ID as part of the query method call. Even though this is redundant because the client object also contains the project ID, we aren't exercising the bigquery.Client constructor in the unit tests to avoid making real API calls. --- packages/pandas-gbq/docs/source/changelog.rst | 2 + packages/pandas-gbq/pandas_gbq/gbq.py | 6 +++ packages/pandas-gbq/tests/unit/test_gbq.py | 51 +++++++++++++++++++ 3 files changed, 59 insertions(+) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index e3c0edd7ad87..723301947121 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -25,6 +25,8 @@ Enhancements - Allow ``table_schema`` in :func:`to_gbq` to contain only a subset of columns, with the rest being populated using the DataFrame dtypes (:issue:`218`) (contributed by @johnpaton) +- Read ``project_id`` in :func:`to_gbq` from provided ``credentials`` if + available (contributed by @daureg) .. _changelog-0.9.0: diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index d0c85a32bf7e..17d18263b4f8 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -318,6 +318,11 @@ def __init__( self.credentials = credentials default_project = None + # Service account credentials have a project associated with them. + # Prefer that project if none was supplied. + if self.project_id is None and hasattr(self.credentials, "project_id"): + self.project_id = credentials.project_id + # Load credentials from cache. if not self.credentials: self.credentials = context.credentials @@ -421,6 +426,7 @@ def run_query(self, query, **kwargs): query, job_config=bigquery.QueryJobConfig.from_api_repr(job_config), location=self.location, + project=self.project_id, ) logger.debug("Query running...") except (RefreshError, ValueError): diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index f4bf8e168f7c..3a0477410b7d 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -47,6 +47,27 @@ def mock_get_credentials(*args, **kwargs): return mock_credentials, "default-project" +@pytest.fixture +def mock_service_account_credentials(): + import google.oauth2.service_account + + mock_credentials = mock.create_autospec( + google.oauth2.service_account.Credentials + ) + return mock_credentials + + +@pytest.fixture +def mock_compute_engine_credentials(): + import google.auth.compute_engine + + mock_credentials = mock.create_autospec( + google.auth.compute_engine.Credentials + ) + return mock_credentials + + +@pytest.fixture def mock_get_user_credentials(*args, **kwargs): import google.auth.credentials @@ -260,6 +281,36 @@ def test_read_gbq_with_inferred_project_id(monkeypatch): assert df is not None +def test_read_gbq_with_inferred_project_id_from_service_account_credentials( + mock_bigquery_client, mock_service_account_credentials +): + mock_service_account_credentials.project_id = "service_account_project_id" + df = gbq.read_gbq( + "SELECT 1", + dialect="standard", + credentials=mock_service_account_credentials, + ) + assert df is not None + mock_bigquery_client.query.assert_called_once_with( + "SELECT 1", + job_config=mock.ANY, + location=None, + project="service_account_project_id", + ) + + +def test_read_gbq_without_inferred_project_id_from_compute_engine_credentials( + mock_compute_engine_credentials +): + with pytest.raises(ValueError) as exception: + gbq.read_gbq( + "SELECT 1", + dialect="standard", + credentials=mock_compute_engine_credentials, + ) + assert "Could not determine project ID" in str(exception) + + def test_read_gbq_with_invalid_private_key_json_should_fail(): with pytest.raises(pandas_gbq.exceptions.InvalidPrivateKeyFormat): gbq.read_gbq( From a1cac5e4d440f84b3299bd8c3ffa80f70e8855b1 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Sat, 23 Mar 2019 16:25:39 -0700 Subject: [PATCH 178/519] DOC: Add meta tag for Google domain verification. --- packages/pandas-gbq/docs/source/index.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/pandas-gbq/docs/source/index.rst b/packages/pandas-gbq/docs/source/index.rst index cbbdabf7289f..e05900b0524d 100644 --- a/packages/pandas-gbq/docs/source/index.rst +++ b/packages/pandas-gbq/docs/source/index.rst @@ -48,3 +48,8 @@ Indices and tables * :ref:`genindex` * :ref:`modindex` * :ref:`search` + +.. Use the meta tags to verify the site for use in Google OAuth2 consent flow. + +.. meta:: + :google-site-verification: 9QSsa9ahOZHbdwZAwl7x-Daaj1W9AttkUOeDgzKtxBw From 3355a59dd48d6c4e4f6daae4802cacce2f79cd85 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 25 Mar 2019 08:50:27 -0700 Subject: [PATCH 179/519] DOC: Add privacy policy for pandas GBQ (#264) This is required for verification of the OAuth 2.0 consent screen. --- .../docs/source/howto/authentication.rst | 1 + packages/pandas-gbq/docs/source/index.rst | 1 + packages/pandas-gbq/docs/source/privacy.rst | 45 +++++++++++++++++++ 3 files changed, 47 insertions(+) create mode 100644 packages/pandas-gbq/docs/source/privacy.rst diff --git a/packages/pandas-gbq/docs/source/howto/authentication.rst b/packages/pandas-gbq/docs/source/howto/authentication.rst index 4612bc4bb0b9..20b0674a2bdc 100644 --- a/packages/pandas-gbq/docs/source/howto/authentication.rst +++ b/packages/pandas-gbq/docs/source/howto/authentication.rst @@ -76,6 +76,7 @@ See the `Getting started with authentication on Google Cloud Platform `_ guide for more information on service accounts. +.. _authentication-user: Authenticating with a User Account ---------------------------------- diff --git a/packages/pandas-gbq/docs/source/index.rst b/packages/pandas-gbq/docs/source/index.rst index e05900b0524d..caadd7954a62 100644 --- a/packages/pandas-gbq/docs/source/index.rst +++ b/packages/pandas-gbq/docs/source/index.rst @@ -40,6 +40,7 @@ Contents: api.rst contributing.rst changelog.rst + privacy.rst Indices and tables diff --git a/packages/pandas-gbq/docs/source/privacy.rst b/packages/pandas-gbq/docs/source/privacy.rst new file mode 100644 index 000000000000..cb59a1a9cce4 --- /dev/null +++ b/packages/pandas-gbq/docs/source/privacy.rst @@ -0,0 +1,45 @@ +Privacy +======= + +This package is a `PyData project `_ and is subject to +the `NumFocus privacy policy `_. Your +use of Google APIs with this module is subject to each API's respective +`terms of service `_. + +Google account and user data +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Accessing user data +~~~~~~~~~~~~~~~~~~~ + +The :mod:`pandas_gbq` module accesses your Google user account, with +the list of `scopes +`_ that you +specify. Depending on your specified list of scopes, the credentials returned +by this library may provide access to other user data, such as your email +address, Google Cloud Platform resources, Google Drive files, or Google +Sheets. + +Storing user data +~~~~~~~~~~~~~~~~~ + +By default, your credentials are stored to a local file, such as +``~/.config/pandas_gbq/bigquery_credentials.dat``. See the +:ref:`authentication-user` guide for details. All user data is stored on +your local machine. **Use caution when using this library on a shared +machine**. + +Sharing user data +~~~~~~~~~~~~~~~~~ + +The pandas-gbq library only communicates with Google APIs. No user +data is shared with PyData, NumFocus, or any other servers. + +Policies for application authors +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Do not use the default client ID when using the pandas-gbq library +from an application, library, or tool. Per the `Google User Data Policy +`_, your +application must accurately represent itself when authenticating to Google +API servcies. From a5cac56158b68139961dae2b60593e802651491e Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 25 Mar 2019 17:03:58 -0700 Subject: [PATCH 180/519] DOC: Clarify data access in the privacy document. --- packages/pandas-gbq/docs/source/privacy.rst | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/packages/pandas-gbq/docs/source/privacy.rst b/packages/pandas-gbq/docs/source/privacy.rst index cb59a1a9cce4..73185182d96d 100644 --- a/packages/pandas-gbq/docs/source/privacy.rst +++ b/packages/pandas-gbq/docs/source/privacy.rst @@ -12,13 +12,16 @@ Google account and user data Accessing user data ~~~~~~~~~~~~~~~~~~~ -The :mod:`pandas_gbq` module accesses your Google user account, with -the list of `scopes -`_ that you -specify. Depending on your specified list of scopes, the credentials returned -by this library may provide access to other user data, such as your email -address, Google Cloud Platform resources, Google Drive files, or Google -Sheets. +The :mod:`pandas_gbq` module accesses Google Cloud Platform resources from +your local machine. Your machine communicates directly with the Google APIs. + +The :func:`~pandas_gbq.read_gbq` function can read and +write BigQuery data (and other data such as Google Sheets or Cloud Storage, +via the federated query feature) through the BigQuery query interface via +queries you supply. + +The :func:`~pandas_gbq.to_gbq` method can write data you supply to a +BigQuery table. Storing user data ~~~~~~~~~~~~~~~~~ From ea7d093c0423aee80cb7c4a028d30ab007bcd1da Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 3 Apr 2019 06:32:16 -0700 Subject: [PATCH 181/519] TST: Use pandas.api.types functions for dtypes checks (#262) * TST: Use pandas.api.types functions for dtypes checks This should make checking for expected dtypes more robust. * Bump minimum pandas version for access to pandas.api.types module. --- packages/pandas-gbq/ci/requirements-2.7.pip | 2 +- packages/pandas-gbq/docs/source/changelog.rst | 3 ++- packages/pandas-gbq/setup.py | 2 +- packages/pandas-gbq/tests/system/test_gbq.py | 19 +++++++++++-------- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/packages/pandas-gbq/ci/requirements-2.7.pip b/packages/pandas-gbq/ci/requirements-2.7.pip index 10300f125329..46c793060510 100644 --- a/packages/pandas-gbq/ci/requirements-2.7.pip +++ b/packages/pandas-gbq/ci/requirements-2.7.pip @@ -1,5 +1,5 @@ mock -pandas==0.17.1 +pandas==0.19.0 google-auth==1.4.1 google-auth-oauthlib==0.0.1 google-cloud-bigquery==1.9.0 diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 723301947121..9e0fbd9de57d 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -13,6 +13,7 @@ Dependency updates - Update the minimum version of ``google-cloud-bigquery`` to 1.9.0. (:issue:`247`) +- Update the minimum version of ``pandas`` to 0.19.0. (:issue:`262`) Internal changes ~~~~~~~~~~~~~~~~ @@ -23,7 +24,7 @@ Internal changes Enhancements ~~~~~~~~~~~~ - Allow ``table_schema`` in :func:`to_gbq` to contain only a subset of columns, - with the rest being populated using the DataFrame dtypes (:issue:`218`) + with the rest being populated using the DataFrame dtypes (:issue:`218`) (contributed by @johnpaton) - Read ``project_id`` in :func:`to_gbq` from provided ``credentials`` if available (contributed by @daureg) diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index e5e40505e38c..8e36e54a6718 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -18,7 +18,7 @@ def readme(): INSTALL_REQUIRES = [ "setuptools", - "pandas", + "pandas>=0.19.0", "pydata-google-auth", "google-auth", "google-auth-oauthlib", diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index a98b679436ab..4480f203d65a 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -7,6 +7,7 @@ import google.oauth2.service_account import numpy as np import pandas +import pandas.api.types import pandas.util.testing as tm from pandas import DataFrame, NaT, compat from pandas.compat import range, u @@ -364,16 +365,18 @@ def test_should_properly_handle_arbitrary_datetime(self, project_id): ) @pytest.mark.parametrize( - "expression, type_", + "expression, is_expected_dtype", [ - ("current_date()", " Date: Wed, 3 Apr 2019 15:47:24 -0700 Subject: [PATCH 182/519] CLN: Update the authentication credentials. (#267) These auth credentials for the browser authentication flow belong to the pandas-gbq-auth GCP project. The reason for the fresh project is that this allows us to explicitly enable APIs needed for new functionality, such as the BigQuery Storage API. --- packages/pandas-gbq/docs/source/changelog.rst | 6 +++++- packages/pandas-gbq/pandas_gbq/auth.py | 9 +++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 9e0fbd9de57d..c2c14cf554f8 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -6,7 +6,11 @@ Changelog 0.10.0 / TBD ------------ -- This fixes a bug where pandas-gbq could not upload an empty database. (:issue:`237`) +- Fix a bug where pandas-gbq could not upload an empty DataFrame. (:issue:`237`) +- Update the authentication credentials. **Note:** You may need to set + ``reauth=True`` in order to update your credentials to the most recent + version. This is required to use new functionality such as the BigQuery + Storage API. (:issue:`267`) Dependency updates ~~~~~~~~~~~~~~~~~~ diff --git a/packages/pandas-gbq/pandas_gbq/auth.py b/packages/pandas-gbq/pandas_gbq/auth.py index b6dca129b4c8..0968dcf08c4b 100644 --- a/packages/pandas-gbq/pandas_gbq/auth.py +++ b/packages/pandas-gbq/pandas_gbq/auth.py @@ -17,8 +17,9 @@ SCOPES = ["https://www.googleapis.com/auth/bigquery"] # The following constants are used for end-user authentication. -# It identifies the application that is requesting permission to access the -# BigQuery API on behalf of a G Suite or Gmail user. +# It identifies (via credentials from the pandas-gbq-auth GCP project) the +# application that is requesting permission to access the BigQuery API on +# behalf of a G Suite or Gmail user. # # In a web application, the client secret would be kept secret, but this is not # possible for applications that are installed locally on an end-user's @@ -26,9 +27,9 @@ # # See: https://cloud.google.com/docs/authentication/end-user for details. CLIENT_ID = ( - "495642085510-k0tmvj2m941jhre2nbqka17vqpjfddtd.apps.googleusercontent.com" + "725825577420-unm2gnkiprugilg743tkbig250f4sfsj.apps.googleusercontent.com" ) -CLIENT_SECRET = "kOc9wMptUtxkcIFbtZCcrEAc" +CLIENT_SECRET = "4hqze9yI8fxShls8eJWkeMdJ" def get_credentials( From 8d7fadb37019b37c40f85fb78725be699a693be9 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 3 Apr 2019 16:13:21 -0700 Subject: [PATCH 183/519] ENH: Use tz-aware dtype for timestamp columns (#269) ENH: Use tz-aware dtype for timestamp columns in all supported pandas versions Adds a table documenting this behavior to the "reading" how-to guides. --- packages/pandas-gbq/docs/source/changelog.rst | 9 +++ packages/pandas-gbq/docs/source/reading.rst | 64 ++++++++++++++----- packages/pandas-gbq/pandas_gbq/gbq.py | 43 ++++++++++--- packages/pandas-gbq/tests/system/test_gbq.py | 37 ++++++----- packages/pandas-gbq/tests/unit/test_gbq.py | 26 ++++++-- 5 files changed, 132 insertions(+), 47 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index c2c14cf554f8..d710b37ff2aa 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -12,6 +12,12 @@ Changelog version. This is required to use new functionality such as the BigQuery Storage API. (:issue:`267`) +Documentation +~~~~~~~~~~~~~ + +- Document :ref:`BigQuery data type to pandas dtype conversion + ` for ``read_gbq``. (:issue:`269`) + Dependency updates ~~~~~~~~~~~~~~~~~~ @@ -27,11 +33,14 @@ Internal changes Enhancements ~~~~~~~~~~~~ + - Allow ``table_schema`` in :func:`to_gbq` to contain only a subset of columns, with the rest being populated using the DataFrame dtypes (:issue:`218`) (contributed by @johnpaton) - Read ``project_id`` in :func:`to_gbq` from provided ``credentials`` if available (contributed by @daureg) +- ``read_gbq`` uses the timezone-aware ``DatetimeTZDtype(unit='ns', + tz='UTC')`` dtype for BigQuery ``TIMESTAMP`` columns. (:issue:`269`) .. _changelog-0.9.0: diff --git a/packages/pandas-gbq/docs/source/reading.rst b/packages/pandas-gbq/docs/source/reading.rst index add61ed27f95..4a7b9d664e96 100644 --- a/packages/pandas-gbq/docs/source/reading.rst +++ b/packages/pandas-gbq/docs/source/reading.rst @@ -9,21 +9,32 @@ Suppose you want to load all data from an existing BigQuery table .. code-block:: python - # Insert your BigQuery Project ID Here - # Can be found in the Google web console + import pandas_gbq + + # TODO: Set your BigQuery Project ID. projectid = "xxxxxxxx" - data_frame = read_gbq('SELECT * FROM test_dataset.test_table', projectid) + data_frame = pandas_gbq.read_gbq( + 'SELECT * FROM `test_dataset.test_table`', + project_id=projectid) + +.. note:: + A project ID is sometimes optional if it can be inferred during + authentication, but it is required when authenticating with user + credentials. You can find your project ID in the `Google Cloud console + `__. You can define which column from BigQuery to use as an index in the destination DataFrame as well as a preferred column order as follows: .. code-block:: python - data_frame = read_gbq('SELECT * FROM test_dataset.test_table', - index_col='index_column_name', - col_order=['col1', 'col2', 'col3'], projectid) + data_frame = pandas_gbq.read_gbq( + 'SELECT * FROM `test_dataset.test_table`', + project_id=projectid, + index_col='index_column_name', + col_order=['col1', 'col2', 'col3']) You can specify the query config as parameter to use additional options of @@ -37,20 +48,39 @@ your job. For more information about query configuration parameters see `here "useQueryCache": False } } - data_frame = read_gbq('SELECT * FROM test_dataset.test_table', - configuration=configuration, projectid) + data_frame = read_gbq( + 'SELECT * FROM `test_dataset.test_table`', + project_id=projectid, + configuration=configuration) -.. note:: +The ``dialect`` argument can be used to indicate whether to use +BigQuery's ``'legacy'`` SQL or BigQuery's ``'standard'`` SQL (beta). The +default value is ``'standard'`` For more information on BigQuery's standard +SQL, see `BigQuery SQL Reference +`__ - You can find your project id in the `Google developers console - `__. +.. code-block:: python + data_frame = pandas_gbq.read_gbq( + 'SELECT * FROM [test_dataset.test_table]', + project_id=projectid, + dialect='legacy') -.. note:: - The ``dialect`` argument can be used to indicate whether to use BigQuery's ``'legacy'`` SQL - or BigQuery's ``'standard'`` SQL (beta). The default value is ``'legacy'``, though this will change - in a subsequent release to ``'standard'``. For more information - on BigQuery's standard SQL, see `BigQuery SQL Reference - `__ +.. _reading-dtypes: + +Inferring the DataFrame's dtypes +-------------------------------- + +The :func:`~pandas_gbq.read_gbq` method infers the pandas dtype for each column, based on the BigQuery table schema. + +================== ========================= +BigQuery Data Type dtype +================== ========================= +FLOAT float +TIMESTAMP :class:`~pandas.DatetimeTZDtype` with ``unit='ns'`` and ``tz='UTC'`` +DATETIME datetime64[ns] +TIME datetime64[ns] +DATE datetime64[ns] +================== ========================= diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 17d18263b4f8..b9978887c347 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -494,6 +494,9 @@ def run_query(self, query, **kwargs): if df.empty: df = _cast_empty_df_dtypes(schema_fields, df) + # Ensure any TIMESTAMP columns are tz-aware. + df = _localize_df(schema_fields, df) + logger.debug("Got {} rows.\n".format(rows_iter.total_rows)) return df @@ -644,17 +647,21 @@ def delete_and_recreate_table(self, dataset_id, table_id, table_schema): def _bqschema_to_nullsafe_dtypes(schema_fields): - # Only specify dtype when the dtype allows nulls. Otherwise, use pandas's - # default dtype choice. - # - # See: - # http://pandas.pydata.org/pandas-docs/dev/missing_data.html - # #missing-data-casting-rules-and-indexing + """Specify explicit dtypes based on BigQuery schema. + + This function only specifies a dtype when the dtype allows nulls. + Otherwise, use pandas's default dtype choice. + + See: http://pandas.pydata.org/pandas-docs/dev/missing_data.html + #missing-data-casting-rules-and-indexing + """ + # If you update this mapping, also update the table at + # `docs/source/reading.rst`. dtype_map = { "FLOAT": np.dtype(float), - # Even though TIMESTAMPs are timezone-aware in BigQuery, pandas doesn't - # support datetime64[ns, UTC] as dtype in DataFrame constructors. See: - # https://github.com/pandas-dev/pandas/issues/12513 + # pandas doesn't support timezone-aware dtype in DataFrame/Series + # constructors. It's more idiomatic to localize after construction. + # https://github.com/pandas-dev/pandas/issues/25843 "TIMESTAMP": "datetime64[ns]", "TIME": "datetime64[ns]", "DATE": "datetime64[ns]", @@ -702,6 +709,24 @@ def _cast_empty_df_dtypes(schema_fields, df): return df +def _localize_df(schema_fields, df): + """Localize any TIMESTAMP columns to tz-aware type. + + In pandas versions before 0.24.0, DatetimeTZDtype cannot be used as the + dtype in Series/DataFrame construction, so localize those columns after + the DataFrame is constructed. + """ + for field in schema_fields: + column = str(field["name"]) + if field["mode"].upper() == "REPEATED": + continue + + if field["type"].upper() == "TIMESTAMP" and df[column].dt.tz is None: + df[column] = df[column].dt.tz_localize("UTC") + + return df + + def read_gbq( query, project_id=None, diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index 4480f203d65a..6c876068d142 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -310,13 +310,15 @@ def test_should_properly_handle_timestamp_unix_epoch(self, project_id): credentials=self.credentials, dialect="legacy", ) - tm.assert_frame_equal( - df, - DataFrame( - {"unix_epoch": ["1970-01-01T00:00:00.000000Z"]}, - dtype="datetime64[ns]", - ), + expected = DataFrame( + {"unix_epoch": ["1970-01-01T00:00:00.000000Z"]}, + dtype="datetime64[ns]", ) + if expected["unix_epoch"].dt.tz is None: + expected["unix_epoch"] = expected["unix_epoch"].dt.tz_localize( + "UTC" + ) + tm.assert_frame_equal(df, expected) def test_should_properly_handle_arbitrary_timestamp(self, project_id): query = 'SELECT TIMESTAMP("2004-09-15 05:00:00") AS valid_timestamp' @@ -326,13 +328,15 @@ def test_should_properly_handle_arbitrary_timestamp(self, project_id): credentials=self.credentials, dialect="legacy", ) - tm.assert_frame_equal( - df, - DataFrame( - {"valid_timestamp": ["2004-09-15T05:00:00.000000Z"]}, - dtype="datetime64[ns]", - ), + expected = DataFrame( + {"valid_timestamp": ["2004-09-15T05:00:00.000000Z"]}, + dtype="datetime64[ns]", ) + if expected["valid_timestamp"].dt.tz is None: + expected["valid_timestamp"] = expected[ + "valid_timestamp" + ].dt.tz_localize("UTC") + tm.assert_frame_equal(df, expected) def test_should_properly_handle_datetime_unix_epoch(self, project_id): query = 'SELECT DATETIME("1970-01-01 00:00:00") AS unix_epoch' @@ -368,7 +372,7 @@ def test_should_properly_handle_arbitrary_datetime(self, project_id): "expression, is_expected_dtype", [ ("current_date()", pandas.api.types.is_datetime64_ns_dtype), - ("current_timestamp()", pandas.api.types.is_datetime64_ns_dtype), + ("current_timestamp()", pandas.api.types.is_datetime64tz_dtype), ("current_datetime()", pandas.api.types.is_datetime64_ns_dtype), ("TRUE", pandas.api.types.is_bool_dtype), ("FALSE", pandas.api.types.is_bool_dtype), @@ -402,9 +406,11 @@ def test_should_properly_handle_null_timestamp(self, project_id): credentials=self.credentials, dialect="legacy", ) - tm.assert_frame_equal( - df, DataFrame({"null_timestamp": [NaT]}, dtype="datetime64[ns]") + expected = DataFrame({"null_timestamp": [NaT]}, dtype="datetime64[ns]") + expected["null_timestamp"] = expected["null_timestamp"].dt.tz_localize( + "UTC" ) + tm.assert_frame_equal(df, expected) def test_should_properly_handle_null_datetime(self, project_id): query = "SELECT CAST(NULL AS DATETIME) AS null_datetime" @@ -594,6 +600,7 @@ def test_zero_rows(self, project_id): expected_result = DataFrame( empty_columns, columns=["title", "id", "is_bot", "ts"] ) + expected_result["ts"] = expected_result["ts"].dt.tz_localize("UTC") tm.assert_frame_equal(df, expected_result, check_index_type=False) def test_one_row_one_column(self, project_id): diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index 3a0477410b7d..6956be20ecde 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -1,21 +1,26 @@ # -*- coding: utf-8 -*- -import pandas.util.testing as tm -import pytest +try: + import mock +except ImportError: # pragma: NO COVER + from unittest import mock + import numpy from pandas import DataFrame +import pandas.util.testing as tm +import pkg_resources +import pytest import pandas_gbq.exceptions from pandas_gbq import gbq -try: - import mock -except ImportError: # pragma: NO COVER - from unittest import mock pytestmark = pytest.mark.filter_warnings( "ignore:credentials from Google Cloud SDK" ) +pandas_installed_version = pkg_resources.get_distribution( + "pandas" +).parsed_version @pytest.fixture @@ -90,6 +95,7 @@ def no_auth(monkeypatch): ("INTEGER", None), # Can't handle NULL ("BOOLEAN", None), # Can't handle NULL ("FLOAT", numpy.dtype(float)), + # TIMESTAMP will be localized after DataFrame construction. ("TIMESTAMP", "datetime64[ns]"), ("DATETIME", "datetime64[ns]"), ], @@ -200,6 +206,10 @@ def test_to_gbq_with_verbose_old_pandas_no_warnings(recwarn, min_bq_version): assert len(recwarn) == 0 +@pytest.mark.skipif( + pandas_installed_version < pkg_resources.parse_version("0.24.0"), + reason="Requires pandas 0.24+", +) def test_to_gbq_with_private_key_new_pandas_warns_deprecation( min_bq_version, monkeypatch ): @@ -413,6 +423,10 @@ def test_read_gbq_with_verbose_old_pandas_no_warnings(recwarn, min_bq_version): assert len(recwarn) == 0 +@pytest.mark.skipif( + pandas_installed_version < pkg_resources.parse_version("0.24.0"), + reason="Requires pandas 0.24+", +) def test_read_gbq_with_private_key_new_pandas_warns_deprecation( min_bq_version, monkeypatch ): From 13e945b0187269f9e3ba90f4f60b8f91ef6f1edf Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 5 Apr 2019 08:13:25 -0500 Subject: [PATCH 184/519] ENH: Add use_bqstorage_api option to read_gbq (#270) * ENH: Add use_bqstorage_api option to read_gbq The BigQuery Storage API provides a way to read query results quickly (and using multiple threads). It only works with large query results (~125 MB), but as of 1.11.1, the google-cloud-bigquery library can fallback to the BigQuery API to download results when a request to the BigQuery Storage API fails. As this API can increase costs (and may not be enabled on the user's project), this option is disabled by default. * Add to changelog. Remove comment about destination tables. * Add docs for using the BigQuery Storage API. --- packages/pandas-gbq/docs/source/changelog.rst | 4 ++ packages/pandas-gbq/docs/source/reading.rst | 38 +++++++++++++ packages/pandas-gbq/pandas_gbq/gbq.py | 53 ++++++++++++++++++- packages/pandas-gbq/tests/system/test_gbq.py | 28 +++++++++- 4 files changed, 121 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index d710b37ff2aa..f18b20d9180d 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -41,6 +41,10 @@ Enhancements available (contributed by @daureg) - ``read_gbq`` uses the timezone-aware ``DatetimeTZDtype(unit='ns', tz='UTC')`` dtype for BigQuery ``TIMESTAMP`` columns. (:issue:`269`) +- Add ``use_bqstorage_api`` to :func:`read_gbq`. The BigQuery Storage API can + be used to download large query results (>125 MB) more quickly. If the BQ + Storage API can't be used, the BigQuery API is used instead. (:issue:`133`, + :issue:`270`) .. _changelog-0.9.0: diff --git a/packages/pandas-gbq/docs/source/reading.rst b/packages/pandas-gbq/docs/source/reading.rst index 4a7b9d664e96..e10f533c3674 100644 --- a/packages/pandas-gbq/docs/source/reading.rst +++ b/packages/pandas-gbq/docs/source/reading.rst @@ -84,3 +84,41 @@ DATETIME datetime64[ns] TIME datetime64[ns] DATE datetime64[ns] ================== ========================= + +.. _reading-bqstorage-api: + +Using the BigQuery Storage API +------------------------------ + +Use the BigQuery Storage API to download large (>125 MB) query results more +quickly (but at an `increased cost +`__) by setting +``use_bqstorage_api`` to ``True``. + +1. Enable the BigQuery Storage API on the project you are using to run + queries. + + `Enable the API + `__. +2. Ensure you have the `*bigquery.readsessions.create permission* + `__. to + create BigQuery Storage API read sessions. This permission is provided by + the `*bigquery.user* role + `__. +4. Install the ``google-cloud-bigquery-storage``, ``fastavro``, and + ``python-snappy`` packages. + + With pip: + + ..code-block:: sh + + pip install --upgrade google-cloud-bigquery-storage fastavro python-snappy + + With conda: + + conda install -c conda-forge google-cloud-bigquery-storage fastavro python-snappy +4. Set ``use_bqstorage_api`` to ``True`` when calling the + :func:`~pandas_gbq.read_gbq` function. As of the ``google-cloud-bigquery`` + package, version 1.11.1 or later,the function will fallback to the + BigQuery API if the BigQuery Storage API cannot be used, such as with + small query results. diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index b9978887c347..9c02538d2e03 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -5,6 +5,13 @@ import numpy as np +try: + # The BigQuery Storage API client is an optional dependency. It is only + # required when use_bqstorage_api=True. + from google.cloud import bigquery_storage_v1beta1 +except ImportError: # pragma: NO COVER + bigquery_storage_v1beta1 = None + from pandas_gbq.exceptions import AccessDenied logger = logging.getLogger(__name__) @@ -302,6 +309,7 @@ def __init__( dialect="standard", location=None, credentials=None, + use_bqstorage_api=False, ): global context from google.api_core.exceptions import GoogleAPIError @@ -352,6 +360,9 @@ def __init__( context.project = self.project_id self.client = self.get_client() + self.bqstorage_client = _make_bqstorage_client( + use_bqstorage_api, self.credentials + ) # BQ Queries costs $5 per TB. First 1 TB per month is free # see here for more: https://cloud.google.com/bigquery/pricing @@ -489,7 +500,9 @@ def run_query(self, query, **kwargs): schema_fields = [field.to_api_repr() for field in rows_iter.schema] nullsafe_dtypes = _bqschema_to_nullsafe_dtypes(schema_fields) - df = rows_iter.to_dataframe(dtypes=nullsafe_dtypes) + df = rows_iter.to_dataframe( + dtypes=nullsafe_dtypes, bqstorage_client=self.bqstorage_client + ) if df.empty: df = _cast_empty_df_dtypes(schema_fields, df) @@ -727,6 +740,21 @@ def _localize_df(schema_fields, df): return df +def _make_bqstorage_client(use_bqstorage_api, credentials): + if not use_bqstorage_api: + return None + + if bigquery_storage_v1beta1 is None: + raise ImportError( + "Install the google-cloud-bigquery-storage and fastavro packages " + "to use the BigQuery Storage API." + ) + + return bigquery_storage_v1beta1.BigQueryStorageClient( + credentials=credentials + ) + + def read_gbq( query, project_id=None, @@ -738,6 +766,7 @@ def read_gbq( location=None, configuration=None, credentials=None, + use_bqstorage_api=False, verbose=None, private_key=None, ): @@ -815,6 +844,27 @@ def read_gbq( :class:`google.oauth2.service_account.Credentials` directly. .. versionadded:: 0.8.0 + use_bqstorage_api : bool, default False + Use the `BigQuery Storage API + `__ to + download query results quickly, but at an increased cost. To use this + API, first `enable it in the Cloud Console + `__. + You must also have the `bigquery.readsessions.create + `__ + permission on the project you are billing queries to. + + **Note:** Due to a `known issue in the ``google-cloud-bigquery`` + package + `__ + (fixed in version 1.11.0), you must write your query results to a + destination table. To do this with ``read_gbq``, supply a + ``configuration`` dictionary. + + This feature requires the ``google-cloud-bigquery-storage`` and + ``fastavro`` packages. + + .. versionadded:: 0.10.0 verbose : None, deprecated Deprecated in Pandas-GBQ 0.4.0. Use the `logging module to adjust verbosity instead @@ -871,6 +921,7 @@ def read_gbq( location=location, credentials=credentials, private_key=private_key, + use_bqstorage_api=use_bqstorage_api, ) final_df = connector.run_query(query, configuration=configuration) diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index 6c876068d142..5c1e6db2a6e3 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -895,10 +895,36 @@ def test_tokyo(self, tokyo_dataset, tokyo_table, private_key_path): location="asia-northeast1", private_key=private_key_path, ) - print(df) assert df["max_year"][0] >= 2000 +@pytest.mark.skip(reason="large query for BQ Storage API tests") +def test_read_gbq_w_bqstorage_api(credentials): + df = gbq.read_gbq( + """ + SELECT + dependency_name, + dependency_platform, + project_name, + project_id, + version_number, + version_id, + dependency_kind, + optional_dependency, + dependency_requirements, + dependency_project_id + FROM + `bigquery-public-data.libraries_io.dependencies` + WHERE + LOWER(dependency_platform) = 'npm' + LIMIT 2500000 + """, + use_bqstorage_api=True, + credentials=credentials, + ) + assert len(df) == 2500000 + + class TestToGBQIntegration(object): @pytest.fixture(autouse=True, scope="function") def setup(self, project, credentials, random_dataset_id): From b6037b83f5abd39659eb79094fe74fe0b6f72921 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 5 Apr 2019 08:21:19 -0500 Subject: [PATCH 185/519] DOC: fix numbering of BQ Storage API steps. --- packages/pandas-gbq/docs/source/reading.rst | 36 +++++++++++---------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/packages/pandas-gbq/docs/source/reading.rst b/packages/pandas-gbq/docs/source/reading.rst index e10f533c3674..0be39b14e6c4 100644 --- a/packages/pandas-gbq/docs/source/reading.rst +++ b/packages/pandas-gbq/docs/source/reading.rst @@ -95,29 +95,31 @@ quickly (but at an `increased cost `__) by setting ``use_bqstorage_api`` to ``True``. -1. Enable the BigQuery Storage API on the project you are using to run - queries. +#. Enable the BigQuery Storage API on the project you are using to run + queries. - `Enable the API - `__. -2. Ensure you have the `*bigquery.readsessions.create permission* - `__. to - create BigQuery Storage API read sessions. This permission is provided by - the `*bigquery.user* role - `__. -4. Install the ``google-cloud-bigquery-storage``, ``fastavro``, and - ``python-snappy`` packages. + `Enable the API + `__. +#. Ensure you have the `bigquery.readsessions.create permission + `__. to + create BigQuery Storage API read sessions. This permission is provided by + the `bigquery.user role + `__. +#. Install the ``google-cloud-bigquery-storage``, ``fastavro``, and + ``python-snappy`` packages. - With pip: + With pip: - ..code-block:: sh + .. code-block:: sh - pip install --upgrade google-cloud-bigquery-storage fastavro python-snappy + pip install --upgrade google-cloud-bigquery-storage fastavro python-snappy - With conda: + With conda: - conda install -c conda-forge google-cloud-bigquery-storage fastavro python-snappy -4. Set ``use_bqstorage_api`` to ``True`` when calling the + .. code-block:: sh + + conda install -c conda-forge google-cloud-bigquery-storage fastavro python-snappy +#. Set ``use_bqstorage_api`` to ``True`` when calling the :func:`~pandas_gbq.read_gbq` function. As of the ``google-cloud-bigquery`` package, version 1.11.1 or later,the function will fallback to the BigQuery API if the BigQuery Storage API cannot be used, such as with From c94d1788f269df9bfb760044782cbd0ac923513e Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 5 Apr 2019 08:26:23 -0500 Subject: [PATCH 186/519] Release 0.10.0 (#272) --- packages/pandas-gbq/docs/source/changelog.rst | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index f18b20d9180d..c6ee70c23bfb 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -3,14 +3,8 @@ Changelog .. _changelog-0.10.0: -0.10.0 / TBD ------------- - -- Fix a bug where pandas-gbq could not upload an empty DataFrame. (:issue:`237`) -- Update the authentication credentials. **Note:** You may need to set - ``reauth=True`` in order to update your credentials to the most recent - version. This is required to use new functionality such as the BigQuery - Storage API. (:issue:`267`) +0.10.0 / 2019-04-05 +------------------- Documentation ~~~~~~~~~~~~~ @@ -28,12 +22,17 @@ Dependency updates Internal changes ~~~~~~~~~~~~~~~~ +- Update the authentication credentials. **Note:** You may need to set + ``reauth=True`` in order to update your credentials to the most recent + version. This is required to use new functionality such as the BigQuery + Storage API. (:issue:`267`) - Use ``to_dataframe()`` from ``google-cloud-bigquery`` in the ``read_gbq()`` function. (:issue:`247`) Enhancements ~~~~~~~~~~~~ +- Fix a bug where pandas-gbq could not upload an empty DataFrame. (:issue:`237`) - Allow ``table_schema`` in :func:`to_gbq` to contain only a subset of columns, with the rest being populated using the DataFrame dtypes (:issue:`218`) (contributed by @johnpaton) From 38f497c40c1189af7041907c8155f69b232b051e Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 19 Apr 2019 14:12:52 -0700 Subject: [PATCH 187/519] CLN: Drop support for Python 2. (#273) The 0.10.0 package will be the latest version to support Python 2. This aligns pandas-gbq with pandas, which dropped support for Python 2 at the end of 2019. --- packages/pandas-gbq/.circleci/config.yml | 9 --------- packages/pandas-gbq/MANIFEST.in | 3 ++- packages/pandas-gbq/ci/requirements-2.7.pip | 6 ------ packages/pandas-gbq/docs/source/changelog.rst | 9 +++++++++ packages/pandas-gbq/noxfile.py | 6 +++--- packages/pandas-gbq/pandas_gbq/auth.py | 6 +----- packages/pandas-gbq/release-procedure.md | 2 +- packages/pandas-gbq/setup.py | 2 -- packages/pandas-gbq/tests/system/test_auth.py | 6 ++---- packages/pandas-gbq/tests/system/test_gbq.py | 12 +++--------- packages/pandas-gbq/tests/unit/conftest.py | 5 +---- packages/pandas-gbq/tests/unit/test_auth.py | 5 +---- packages/pandas-gbq/tests/unit/test_context.py | 5 +---- packages/pandas-gbq/tests/unit/test_gbq.py | 5 +---- 14 files changed, 25 insertions(+), 56 deletions(-) delete mode 100644 packages/pandas-gbq/ci/requirements-2.7.pip diff --git a/packages/pandas-gbq/.circleci/config.yml b/packages/pandas-gbq/.circleci/config.yml index 4a88a1827b11..bd9574939817 100644 --- a/packages/pandas-gbq/.circleci/config.yml +++ b/packages/pandas-gbq/.circleci/config.yml @@ -1,14 +1,6 @@ version: 2 jobs: # Pip - "pip-2.7": - docker: - - image: thekevjames/nox - steps: - - checkout - - run: ci/config_auth.sh - - run: nox -s unit-2.7 system-2.7 - "pip-3.5": docker: - image: thekevjames/nox @@ -60,7 +52,6 @@ workflows: version: 2 build: jobs: - - "pip-2.7" - "pip-3.5" - "pip-3.6" - "pip-3.7" diff --git a/packages/pandas-gbq/MANIFEST.in b/packages/pandas-gbq/MANIFEST.in index 273fd19a980f..c27fd5f3d581 100644 --- a/packages/pandas-gbq/MANIFEST.in +++ b/packages/pandas-gbq/MANIFEST.in @@ -1,6 +1,7 @@ include MANIFEST.in include README.rst -include LICENSE.md +include AUTHORS.md +include LICENSE.txt include setup.py graft pandas_gbq diff --git a/packages/pandas-gbq/ci/requirements-2.7.pip b/packages/pandas-gbq/ci/requirements-2.7.pip deleted file mode 100644 index 46c793060510..000000000000 --- a/packages/pandas-gbq/ci/requirements-2.7.pip +++ /dev/null @@ -1,6 +0,0 @@ -mock -pandas==0.19.0 -google-auth==1.4.1 -google-auth-oauthlib==0.0.1 -google-cloud-bigquery==1.9.0 -pydata-google-auth==0.1.2 diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index c6ee70c23bfb..8cdd5c8aa07e 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,6 +1,15 @@ Changelog ========= +.. _changelog-0.11.0: + +0.11.0 / TBD +------------ + +- **Breaking Change:** Python 2 support has been dropped. This is to align + with the pandas package which dropped Python 2 support at the end of 2019. + (:issue:`268`) + .. _changelog-0.10.0: 0.10.0 / 2019-04-05 diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index c4c8a3c00382..d13a732b4d33 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -10,8 +10,8 @@ import nox -supported_pythons = ["2.7", "3.5", "3.6", "3.7"] -latest_python = "3.6" +supported_pythons = ["3.5", "3.6", "3.7"] +latest_python = "3.7" @nox.session @@ -31,7 +31,7 @@ def blacken(session): @nox.session(python=supported_pythons) def unit(session): - session.install("mock", "pytest", "pytest-cov") + session.install("pytest", "pytest-cov") session.install("-e", ".") session.run( "pytest", diff --git a/packages/pandas-gbq/pandas_gbq/auth.py b/packages/pandas-gbq/pandas_gbq/auth.py index 0968dcf08c4b..bd842d086bdc 100644 --- a/packages/pandas-gbq/pandas_gbq/auth.py +++ b/packages/pandas-gbq/pandas_gbq/auth.py @@ -5,8 +5,6 @@ import os import os.path -import pandas.compat - import pandas_gbq.exceptions logger = logging.getLogger(__name__) @@ -72,9 +70,7 @@ def get_service_account_credentials(private_key): " ", "\n" ) - if pandas.compat.PY3: - json_key["private_key"] = bytes(json_key["private_key"], "UTF-8") - + json_key["private_key"] = bytes(json_key["private_key"], "UTF-8") credentials = Credentials.from_service_account_info(json_key) credentials = credentials.with_scopes(SCOPES) diff --git a/packages/pandas-gbq/release-procedure.md b/packages/pandas-gbq/release-procedure.md index 58dfda9b59dd..b682db2fa3c6 100644 --- a/packages/pandas-gbq/release-procedure.md +++ b/packages/pandas-gbq/release-procedure.md @@ -21,7 +21,7 @@ * Build the package git clean -xfd - python setup.py register sdist bdist_wheel --universal + python setup.py register sdist bdist_wheel * Upload to test PyPI diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index 8e36e54a6718..f9a0a4319db9 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -43,8 +43,6 @@ def readme(): "Intended Audience :: Science/Research", "Operating System :: OS Independent", "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", diff --git a/packages/pandas-gbq/tests/system/test_auth.py b/packages/pandas-gbq/tests/system/test_auth.py index d02f153a365e..980fbb48ce2e 100644 --- a/packages/pandas-gbq/tests/system/test_auth.py +++ b/packages/pandas-gbq/tests/system/test_auth.py @@ -1,9 +1,7 @@ """System tests for fetching Google BigQuery credentials.""" -try: - import mock -except ImportError: # pragma: NO COVER - from unittest import mock +from unittest import mock + import pytest from pandas_gbq import auth diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index 5c1e6db2a6e3..a6f371af4a82 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -9,8 +9,7 @@ import pandas import pandas.api.types import pandas.util.testing as tm -from pandas import DataFrame, NaT, compat -from pandas.compat import range, u +from pandas import DataFrame, NaT import pytest import pytz @@ -447,13 +446,8 @@ def test_should_properly_handle_nullable_booleans(self, project_id): ) def test_unicode_string_conversion_and_normalization(self, project_id): - correct_test_datatype = DataFrame({"unicode_string": [u("\xe9\xfc")]}) - - unicode_string = "\xc3\xa9\xc3\xbc" - - if compat.PY3: - unicode_string = unicode_string.encode("latin-1").decode("utf8") - + correct_test_datatype = DataFrame({"unicode_string": ["éü"]}) + unicode_string = "éü" query = 'SELECT "{0}" AS unicode_string'.format(unicode_string) df = gbq.read_gbq( diff --git a/packages/pandas-gbq/tests/unit/conftest.py b/packages/pandas-gbq/tests/unit/conftest.py index ece8421d933b..c1d9e0f8a362 100644 --- a/packages/pandas-gbq/tests/unit/conftest.py +++ b/packages/pandas-gbq/tests/unit/conftest.py @@ -1,9 +1,6 @@ # -*- coding: utf-8 -*- -try: - from unittest import mock -except ImportError: # pragma: NO COVER - import mock +from unittest import mock import pytest diff --git a/packages/pandas-gbq/tests/unit/test_auth.py b/packages/pandas-gbq/tests/unit/test_auth.py index f9f0dc940d91..fa054fe1efbe 100644 --- a/packages/pandas-gbq/tests/unit/test_auth.py +++ b/packages/pandas-gbq/tests/unit/test_auth.py @@ -5,10 +5,7 @@ from pandas_gbq import auth -try: - import mock -except ImportError: # pragma: NO COVER - from unittest import mock +from unittest import mock def test_get_credentials_private_key_contents(monkeypatch): diff --git a/packages/pandas-gbq/tests/unit/test_context.py b/packages/pandas-gbq/tests/unit/test_context.py index 59a91501c208..cb0961106329 100644 --- a/packages/pandas-gbq/tests/unit/test_context.py +++ b/packages/pandas-gbq/tests/unit/test_context.py @@ -1,9 +1,6 @@ # -*- coding: utf-8 -*- -try: - from unittest import mock -except ImportError: # pragma: NO COVER - import mock +from unittest import mock import pytest diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index 6956be20ecde..bf609385ce5b 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -1,9 +1,6 @@ # -*- coding: utf-8 -*- -try: - import mock -except ImportError: # pragma: NO COVER - from unittest import mock +from unittest import mock import numpy from pandas import DataFrame From c3a895428ab98bc52735dcefbf7ab594df84da47 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 10 May 2019 10:15:39 -0700 Subject: [PATCH 188/519] TST: Update Conda CI build to use nightly pandas packages. (#254) * TST: Update Conda CI build to use nightly pandas packages. Remove unused git-based pip requirements file. * Restore non-nightly conda build. --- packages/pandas-gbq/.circleci/config.yml | 13 ++++++++++++- packages/pandas-gbq/ci/requirements-3.6-MASTER.pip | 5 ----- .../pandas-gbq/ci/requirements-3.7-NIGHTLY.conda | 7 +++++++ packages/pandas-gbq/ci/run_conda.sh | 2 +- 4 files changed, 20 insertions(+), 7 deletions(-) delete mode 100644 packages/pandas-gbq/ci/requirements-3.6-MASTER.pip create mode 100644 packages/pandas-gbq/ci/requirements-3.7-NIGHTLY.conda diff --git a/packages/pandas-gbq/.circleci/config.yml b/packages/pandas-gbq/.circleci/config.yml index bd9574939817..053280cddf1c 100644 --- a/packages/pandas-gbq/.circleci/config.yml +++ b/packages/pandas-gbq/.circleci/config.yml @@ -47,6 +47,16 @@ jobs: - checkout - run: ci/config_auth.sh - run: ci/run_conda.sh + "conda-3.7-NIGHTLY": + docker: + - image: continuumio/miniconda3 + environment: + PYTHON: "3.7" + PANDAS: "NIGHTLY" + steps: + - checkout + - run: ci/config_auth.sh + - run: ci/run_conda.sh workflows: version: 2 @@ -56,4 +66,5 @@ workflows: - "pip-3.6" - "pip-3.7" - lint - - "conda-3.6-0.20.1" \ No newline at end of file + - "conda-3.6-0.20.1" + - "conda-3.7-NIGHTLY" \ No newline at end of file diff --git a/packages/pandas-gbq/ci/requirements-3.6-MASTER.pip b/packages/pandas-gbq/ci/requirements-3.6-MASTER.pip deleted file mode 100644 index 7f44634287b7..000000000000 --- a/packages/pandas-gbq/ci/requirements-3.6-MASTER.pip +++ /dev/null @@ -1,5 +0,0 @@ ---pre -f https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com/ pandas -git+https://github.com/googleapis/google-cloud-python.git#egg=version_subpkg&subdirectory=api_core -git+https://github.com/googleapis/google-cloud-python.git#egg=version_subpkg&subdirectory=core -git+https://github.com/googleapis/google-cloud-python.git#egg=version_subpkg&subdirectory=bigquery -pydata-google-auth==0.1.2 \ No newline at end of file diff --git a/packages/pandas-gbq/ci/requirements-3.7-NIGHTLY.conda b/packages/pandas-gbq/ci/requirements-3.7-NIGHTLY.conda new file mode 100644 index 000000000000..419abbad0d7f --- /dev/null +++ b/packages/pandas-gbq/ci/requirements-3.7-NIGHTLY.conda @@ -0,0 +1,7 @@ +pydata-google-auth +google-cloud-bigquery==1.10.0 +pytest +pytest-cov +codecov +coverage +flake8 diff --git a/packages/pandas-gbq/ci/run_conda.sh b/packages/pandas-gbq/ci/run_conda.sh index 60ae6ff0f079..d0b892aac490 100755 --- a/packages/pandas-gbq/ci/run_conda.sh +++ b/packages/pandas-gbq/ci/run_conda.sh @@ -11,7 +11,7 @@ conda update -q conda conda info -a conda create -q -n test-environment python=$PYTHON source activate test-environment -if [[ "$PANDAS" == "MASTER" ]]; then +if [[ "$PANDAS" == "NIGHTLY" ]]; then conda install -q numpy pytz python-dateutil; PRE_WHEELS="https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com"; pip install --pre --upgrade --timeout=60 -f $PRE_WHEELS pandas; From 5cd78e87e95c1efc56d2c738da9dc3e557ec7425 Mon Sep 17 00:00:00 2001 From: Brad Date: Wed, 29 May 2019 13:13:43 -0400 Subject: [PATCH 189/519] BUG: Ensure table_schema arg not modified inplace (#278) Fixes #277. If any dictionary entry in the `table_schema` arg did not contain a "mode" key, a mode="NULLABLE" kv pair would be created; because `schema.update_schema()` returns an object with references to its mutable input, this allows the argument to ultimately be modified by the function rather than the caller. This pattern was used in both gbq.py and load.py, so it was refactored into a helper function in schema.py, which now returns a modified *copy*. Deepcopy is used because the input is a list of dictionaries, so a shallow copy would be insufficient. --- packages/pandas-gbq/pandas_gbq/gbq.py | 8 +--- packages/pandas-gbq/pandas_gbq/load.py | 7 +--- packages/pandas-gbq/pandas_gbq/schema.py | 15 ++++++++ packages/pandas-gbq/tests/unit/test_gbq.py | 44 ++++++++++++++++++++++ 4 files changed, 62 insertions(+), 12 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 9c02538d2e03..4ec7f80451a4 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -13,6 +13,7 @@ bigquery_storage_v1beta1 = None from pandas_gbq.exceptions import AccessDenied +import pandas_gbq.schema logger = logging.getLogger(__name__) @@ -1269,12 +1270,7 @@ def create(self, table_id, schema): table_ref = self.client.dataset(self.dataset_id).table(table_id) table = Table(table_ref) - # Manually create the schema objects, adding NULLABLE mode - # as a workaround for - # https://github.com/GoogleCloudPlatform/google-cloud-python/issues/4456 - for field in schema["fields"]: - if "mode" not in field: - field["mode"] = "NULLABLE" + schema = pandas_gbq.schema.add_default_nullable_mode(schema) table.schema = [ SchemaField.from_api_repr(field) for field in schema["fields"] diff --git a/packages/pandas-gbq/pandas_gbq/load.py b/packages/pandas-gbq/pandas_gbq/load.py index cb190f865351..3e9d570e6e85 100644 --- a/packages/pandas-gbq/pandas_gbq/load.py +++ b/packages/pandas-gbq/pandas_gbq/load.py @@ -66,12 +66,7 @@ def load_chunks( if schema is None: schema = pandas_gbq.schema.generate_bq_schema(dataframe) - # Manually create the schema objects, adding NULLABLE mode - # as a workaround for - # https://github.com/GoogleCloudPlatform/google-cloud-python/issues/4456 - for field in schema["fields"]: - if "mode" not in field: - field["mode"] = "NULLABLE" + schema = pandas_gbq.schema.add_default_nullable_mode(schema) job_config.schema = [ bigquery.SchemaField.from_api_repr(field) for field in schema["fields"] diff --git a/packages/pandas-gbq/pandas_gbq/schema.py b/packages/pandas-gbq/pandas_gbq/schema.py index 91963b7cca0c..bb18fabc6a1c 100644 --- a/packages/pandas-gbq/pandas_gbq/schema.py +++ b/packages/pandas-gbq/pandas_gbq/schema.py @@ -1,5 +1,7 @@ """Helper methods for BigQuery schemas""" +import copy + def generate_bq_schema(dataframe, default_type="STRING"): """Given a passed dataframe, generate the associated Google BigQuery schema. @@ -62,3 +64,16 @@ def update_schema(schema_old, schema_new): output_fields.append(field) return {"fields": output_fields} + + +def add_default_nullable_mode(schema): + """Manually create the schema objects, adding NULLABLE mode.""" + # Workaround for: + # https://github.com/GoogleCloudPlatform/google-cloud-python/issues/4456 + # + # Returns a copy rather than modifying the mutable arg, + # per Issue #277 + result = copy.deepcopy(schema) + for field in result["fields"]: + field.setdefault("mode", "NULLABLE") + return result diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index bf609385ce5b..f2e524627f9d 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +import copy +import datetime from unittest import mock import numpy @@ -477,3 +479,45 @@ def test_generate_bq_schema_deprecated(): with pytest.warns(FutureWarning): df = DataFrame([[1, "two"], [3, "four"]]) gbq.generate_bq_schema(df) + + +def test_load_does_not_modify_schema_arg(): + # Test of Issue # 277 + df = DataFrame( + { + "field1": ["a", "b"], + "field2": [1, 2], + "field3": [datetime.date(2019, 1, 1), datetime.date(2019, 5, 1)], + } + ) + original_schema = [ + {"name": "field1", "type": "STRING", "mode": "REQUIRED"}, + {"name": "field2", "type": "INTEGER"}, + {"name": "field3", "type": "DATE"}, + ] + original_schema_cp = copy.deepcopy(original_schema) + gbq.to_gbq( + df, + "dataset.schematest", + project_id="my-project", + table_schema=original_schema, + if_exists="fail", + ) + assert original_schema == original_schema_cp + + # Test again now that table exists - behavior will differ internally + # branch at if table.exists(table_id) + original_schema = [ + {"name": "field1", "type": "STRING", "mode": "REQUIRED"}, + {"name": "field2", "type": "INTEGER"}, + {"name": "field3", "type": "DATE"}, + ] + original_schema_cp = copy.deepcopy(original_schema) + gbq.to_gbq( + df, + "dataset.schematest", + project_id="my-project", + table_schema=original_schema, + if_exists="append", + ) + assert original_schema == original_schema_cp From 52242f8fa4a0608f6b7e5ac67770ca0ba78c4240 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 22 Jul 2019 13:59:34 -0700 Subject: [PATCH 190/519] TST: Fix pytest.raises usage for latest pytest. Fix warnings in tests. (#282) Tests were failing because the `str` method for the context returned by `pytest.raises` no longer prints the contained exception. Instead, use `match=regex_value` to check for the desired error message. --- packages/pandas-gbq/tests/system/test_gbq.py | 23 +++++++++++++++----- packages/pandas-gbq/tests/unit/test_gbq.py | 12 ++++------ 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index a6f371af4a82..333695573415 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -868,26 +868,27 @@ def test_array_agg(self, project_id): ), ) - def test_array_of_floats(self, private_key_path, project_id): + def test_array_of_floats(self, project_id): query = """select [1.1, 2.2, 3.3] as a, 4 as b""" df = gbq.read_gbq( query, project_id=project_id, - private_key=private_key_path, + credentials=self.credentials, dialect="standard", ) tm.assert_frame_equal( df, DataFrame([[[1.1, 2.2, 3.3], 4]], columns=["a", "b"]) ) - def test_tokyo(self, tokyo_dataset, tokyo_table, private_key_path): + def test_tokyo(self, tokyo_dataset, tokyo_table, project_id): df = gbq.read_gbq( "SELECT MAX(year) AS max_year FROM {}.{}".format( tokyo_dataset, tokyo_table ), dialect="standard", location="asia-northeast1", - private_key=private_key_path, + project_id=project_id, + credentials=self.credentials, ) assert df["max_year"][0] >= 2000 @@ -1401,7 +1402,17 @@ def test_upload_data_with_timestamp(self, project_id): index=range(test_size), columns=list("ABCD"), ) - df["times"] = np.datetime64("2018-03-13T05:40:45.348318Z") + df["times"] = pandas.Series( + [ + "2018-03-13T05:40:45.348318", + "2018-04-13T05:40:45.348318", + "2018-05-13T05:40:45.348318", + "2018-06-13T05:40:45.348318", + "2018-07-13T05:40:45.348318", + "2018-08-13T05:40:45.348318", + ], + dtype="datetime64[ns]", + ).dt.tz_localize("UTC") gbq.to_gbq( df, @@ -1421,7 +1432,7 @@ def test_upload_data_with_timestamp(self, project_id): expected = df["times"].sort_values() result = result_df["times"].sort_values() - tm.assert_numpy_array_equal(expected.values, result.values) + tm.assert_series_equal(expected, result) def test_upload_data_with_different_df_and_user_schema(self, project_id): df = tm.makeMixedDataFrame() diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index f2e524627f9d..0452bf8882b4 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -121,9 +121,8 @@ def test_to_gbq_with_no_project_id_given_should_fail(monkeypatch): pydata_google_auth, "default", mock_get_credentials_no_project ) - with pytest.raises(ValueError) as exception: + with pytest.raises(ValueError, match="Could not determine project ID"): gbq.to_gbq(DataFrame([[1]]), "dataset.tablename") - assert "Could not determine project ID" in str(exception) def test_to_gbq_with_verbose_new_pandas_warns_deprecation(min_bq_version): @@ -280,9 +279,8 @@ def test_read_gbq_with_no_project_id_given_should_fail(monkeypatch): pydata_google_auth, "default", mock_get_credentials_no_project ) - with pytest.raises(ValueError) as exception: + with pytest.raises(ValueError, match="Could not determine project ID"): gbq.read_gbq("SELECT 1", dialect="standard") - assert "Could not determine project ID" in str(exception) def test_read_gbq_with_inferred_project_id(monkeypatch): @@ -311,13 +309,12 @@ def test_read_gbq_with_inferred_project_id_from_service_account_credentials( def test_read_gbq_without_inferred_project_id_from_compute_engine_credentials( mock_compute_engine_credentials ): - with pytest.raises(ValueError) as exception: + with pytest.raises(ValueError, match="Could not determine project ID"): gbq.read_gbq( "SELECT 1", dialect="standard", credentials=mock_compute_engine_credentials, ) - assert "Could not determine project ID" in str(exception) def test_read_gbq_with_invalid_private_key_json_should_fail(): @@ -469,9 +466,8 @@ def test_read_gbq_with_private_key_old_pandas_no_warnings( def test_read_gbq_with_invalid_dialect(): - with pytest.raises(ValueError) as excinfo: + with pytest.raises(ValueError, match="is not valid for dialect"): gbq.read_gbq("SELECT 1", dialect="invalid") - assert "is not valid for dialect" in str(excinfo.value) def test_generate_bq_schema_deprecated(): From dbf487b897a22a5131ecf38ecef5efca51005088 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 26 Jul 2019 11:29:57 -0700 Subject: [PATCH 191/519] BUG: Use object dtype for STRING, ARRAY, and STRUCT columns when there are zero rows. (#285) * BUG: Use object dtype for STRING, ARRAY, and STRUCT columns when there are zero rows. If a there are no rows, the default dtype is used (which is now float64, must previously have been object). * Add PR number to changelog. * Blacken --- packages/pandas-gbq/docs/source/changelog.rst | 6 +++++ packages/pandas-gbq/pandas_gbq/gbq.py | 12 +++++++--- packages/pandas-gbq/tests/system/test_gbq.py | 23 ++++++++++--------- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 8cdd5c8aa07e..bdf4413bc63c 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -10,6 +10,12 @@ Changelog with the pandas package which dropped Python 2 support at the end of 2019. (:issue:`268`) +Implementation changes +~~~~~~~~~~~~~~~~~~~~~~ + +- Use object dtype for ``STRING``, ``ARRAY``, and ``STRUCT`` columns when + there are zero rows. (:issue:`285`) + .. _changelog-0.10.0: 0.10.0 / 2019-04-05 diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 4ec7f80451a4..4590e1daefa8 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -672,20 +672,26 @@ def _bqschema_to_nullsafe_dtypes(schema_fields): # If you update this mapping, also update the table at # `docs/source/reading.rst`. dtype_map = { + "DATE": "datetime64[ns]", + "DATETIME": "datetime64[ns]", "FLOAT": np.dtype(float), + "GEOMETRY": "object", + "RECORD": "object", + "STRING": "object", + "TIME": "datetime64[ns]", # pandas doesn't support timezone-aware dtype in DataFrame/Series # constructors. It's more idiomatic to localize after construction. # https://github.com/pandas-dev/pandas/issues/25843 "TIMESTAMP": "datetime64[ns]", - "TIME": "datetime64[ns]", - "DATE": "datetime64[ns]", - "DATETIME": "datetime64[ns]", } dtypes = {} for field in schema_fields: name = str(field["name"]) + # Array BigQuery type is represented as an object column containing + # list objects. if field["mode"].upper() == "REPEATED": + dtypes[name] = "object" continue dtype = dtype_map.get(field["type"].upper()) diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index 333695573415..6f8ef406c622 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -577,24 +577,25 @@ def test_download_dataset_larger_than_200k_rows(self, project_id): def test_zero_rows(self, project_id): # Bug fix for https://github.com/pandas-dev/pandas/issues/10273 df = gbq.read_gbq( - "SELECT title, id, is_bot, " - "SEC_TO_TIMESTAMP(timestamp) ts " - "FROM [publicdata:samples.wikipedia] " - "WHERE timestamp=-9999999", + 'SELECT name, number, (mlc_class = "HU") is_hurricane, iso_time ' + "FROM `bigquery-public-data.noaa_hurricanes.hurricanes` " + 'WHERE iso_time = TIMESTAMP("1900-01-01 00:00:00") ', project_id=project_id, credentials=self.credentials, - dialect="legacy", ) empty_columns = { - "title": pandas.Series([], dtype=object), - "id": pandas.Series([], dtype=np.dtype(int)), - "is_bot": pandas.Series([], dtype=np.dtype(bool)), - "ts": pandas.Series([], dtype="datetime64[ns]"), + "name": pandas.Series([], dtype=object), + "number": pandas.Series([], dtype=np.dtype(int)), + "is_hurricane": pandas.Series([], dtype=np.dtype(bool)), + "iso_time": pandas.Series([], dtype="datetime64[ns]"), } expected_result = DataFrame( - empty_columns, columns=["title", "id", "is_bot", "ts"] + empty_columns, + columns=["name", "number", "is_hurricane", "iso_time"], ) - expected_result["ts"] = expected_result["ts"].dt.tz_localize("UTC") + expected_result["iso_time"] = expected_result[ + "iso_time" + ].dt.tz_localize("UTC") tm.assert_frame_equal(df, expected_result, check_index_type=False) def test_one_row_one_column(self, project_id): From dd9c90a2ef8b65b373f043bc84213793c09d9988 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 29 Jul 2019 09:39:20 -0700 Subject: [PATCH 192/519] ENH: Add user-agent string when constructing BigQuery and BigQuery Storage API clients. (#284) * ENH: Add user-agent string when constructing BigQuery and BigQuery Storage API clients. Since this was a relatively new addition to the BigQuery client library, only populate the user-agent for BigQuery when recent-enough versions of google-cloud-bigquery and google-api-core are installed. * Add google-cloud-bigquery-storage to conda tests. * Skip BigQuery Storage API test when package not available. Use a smaller query to test the API. * Make the BQ Storage API test query slightly smaller. * Add fastavro to conda deps. --- .../ci/requirements-3.6-0.20.1.conda | 10 ++-- .../ci/requirements-3.7-NIGHTLY.conda | 3 +- packages/pandas-gbq/ci/requirements-3.7.pip | 2 +- packages/pandas-gbq/docs/source/changelog.rst | 5 ++ packages/pandas-gbq/pandas_gbq/gbq.py | 54 +++++++++++++++--- packages/pandas-gbq/tests/system/test_gbq.py | 56 +++++++++++++------ packages/pandas-gbq/tests/unit/conftest.py | 7 ++- packages/pandas-gbq/tests/unit/test_auth.py | 2 - packages/pandas-gbq/tests/unit/test_gbq.py | 36 +++++++++++- 9 files changed, 138 insertions(+), 37 deletions(-) diff --git a/packages/pandas-gbq/ci/requirements-3.6-0.20.1.conda b/packages/pandas-gbq/ci/requirements-3.6-0.20.1.conda index 1c7eb3f22b84..e492c378eb1a 100644 --- a/packages/pandas-gbq/ci/requirements-3.6-0.20.1.conda +++ b/packages/pandas-gbq/ci/requirements-3.6-0.20.1.conda @@ -1,7 +1,9 @@ -pydata-google-auth -google-cloud-bigquery==1.9.0 -pytest -pytest-cov codecov coverage +fastavro flake8 +google-cloud-bigquery==1.9.0 +google-cloud-bigquery-storage==0.5.0 +pydata-google-auth +pytest +pytest-cov diff --git a/packages/pandas-gbq/ci/requirements-3.7-NIGHTLY.conda b/packages/pandas-gbq/ci/requirements-3.7-NIGHTLY.conda index 419abbad0d7f..9dfe3f6b071f 100644 --- a/packages/pandas-gbq/ci/requirements-3.7-NIGHTLY.conda +++ b/packages/pandas-gbq/ci/requirements-3.7-NIGHTLY.conda @@ -1,5 +1,6 @@ pydata-google-auth -google-cloud-bigquery==1.10.0 +google-cloud-bigquery +google-cloud-bigquery-storage pytest pytest-cov codecov diff --git a/packages/pandas-gbq/ci/requirements-3.7.pip b/packages/pandas-gbq/ci/requirements-3.7.pip index 710cb8b8a01e..6025ac8a4c4d 100644 --- a/packages/pandas-gbq/ci/requirements-3.7.pip +++ b/packages/pandas-gbq/ci/requirements-3.7.pip @@ -1,3 +1,3 @@ pandas==0.24.0 -google-cloud-bigquery==1.9.0 +google-cloud-bigquery==1.12.0 pydata-google-auth==0.1.2 \ No newline at end of file diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index bdf4413bc63c..5e77de5d600a 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -16,6 +16,11 @@ Implementation changes - Use object dtype for ``STRING``, ``ARRAY``, and ``STRUCT`` columns when there are zero rows. (:issue:`285`) +Internal changes +~~~~~~~~~~~~~~~~ + +- Populate ``user-agent`` with ``pandas`` version information. (:issue:`281`) + .. _changelog-0.10.0: 0.10.0 / 2019-04-05 diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 4590e1daefa8..507432db0674 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -8,9 +8,9 @@ try: # The BigQuery Storage API client is an optional dependency. It is only # required when use_bqstorage_api=True. - from google.cloud import bigquery_storage_v1beta1 + from google.cloud import bigquery_storage except ImportError: # pragma: NO COVER - bigquery_storage_v1beta1 = None + bigquery_storage = None from pandas_gbq.exceptions import AccessDenied import pandas_gbq.schema @@ -18,6 +18,8 @@ logger = logging.getLogger(__name__) BIGQUERY_INSTALLED_VERSION = None +BIGQUERY_CLIENT_INFO_VERSION = "1.12.0" +HAS_CLIENT_INFO = False SHOW_VERBOSE_DEPRECATION = False SHOW_PRIVATE_KEY_DEPRECATION = False PRIVATE_KEY_DEPRECATION_MESSAGE = ( @@ -34,7 +36,7 @@ def _check_google_client_version(): - global BIGQUERY_INSTALLED_VERSION, SHOW_VERBOSE_DEPRECATION, SHOW_PRIVATE_KEY_DEPRECATION + global BIGQUERY_INSTALLED_VERSION, HAS_CLIENT_INFO, SHOW_VERBOSE_DEPRECATION, SHOW_PRIVATE_KEY_DEPRECATION try: import pkg_resources @@ -44,10 +46,17 @@ def _check_google_client_version(): # https://github.com/GoogleCloudPlatform/google-cloud-python/blob/master/bigquery/CHANGELOG.md bigquery_minimum_version = pkg_resources.parse_version("1.9.0") + bigquery_client_info_version = pkg_resources.parse_version( + BIGQUERY_CLIENT_INFO_VERSION + ) BIGQUERY_INSTALLED_VERSION = pkg_resources.get_distribution( "google-cloud-bigquery" ).parsed_version + HAS_CLIENT_INFO = ( + BIGQUERY_INSTALLED_VERSION >= bigquery_client_info_version + ) + if BIGQUERY_INSTALLED_VERSION < bigquery_minimum_version: raise ImportError( "pandas-gbq requires google-cloud-bigquery >= {0}, " @@ -392,6 +401,29 @@ def sizeof_fmt(num, suffix="B"): def get_client(self): from google.cloud import bigquery + import pandas + + try: + # This module was added in google-api-core 1.11.0. + # We don't have a hard requirement on that version, so only + # populate the client_info if available. + import google.api_core.client_info + + client_info = google.api_core.client_info.ClientInfo( + user_agent="pandas-{}".format(pandas.__version__) + ) + except ImportError: + client_info = None + + # In addition to new enough version of google-api-core, a new enough + # version of google-cloud-bigquery is required to populate the + # client_info. + if HAS_CLIENT_INFO: + return bigquery.Client( + project=self.project_id, + credentials=self.credentials, + client_info=client_info, + ) return bigquery.Client( project=self.project_id, credentials=self.credentials @@ -751,14 +783,20 @@ def _make_bqstorage_client(use_bqstorage_api, credentials): if not use_bqstorage_api: return None - if bigquery_storage_v1beta1 is None: + if bigquery_storage is None: raise ImportError( - "Install the google-cloud-bigquery-storage and fastavro packages " - "to use the BigQuery Storage API." + "Install the google-cloud-bigquery-storage and fastavro/pyarrow " + "packages to use the BigQuery Storage API." ) - return bigquery_storage_v1beta1.BigQueryStorageClient( - credentials=credentials + import google.api_core.gapic_v1.client_info + import pandas + + client_info = google.api_core.gapic_v1.client_info.ClientInfo( + user_agent="pandas-{}".format(pandas.__version__) + ) + return bigquery_storage.BigQueryStorageClient( + credentials=credentials, client_info=client_info ) diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index 6f8ef406c622..ebe7782c3138 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -70,6 +70,16 @@ def random_dataset_id(bigquery_client): pass # Not all tests actually create a dataset +@pytest.fixture() +def random_dataset(bigquery_client, random_dataset_id): + from google.cloud import bigquery + + dataset_ref = bigquery_client.dataset(random_dataset_id) + dataset = bigquery.Dataset(dataset_ref) + bigquery_client.create_dataset(dataset) + return dataset + + @pytest.fixture() def tokyo_dataset(bigquery_client, random_dataset_id): from google.cloud import bigquery @@ -894,31 +904,41 @@ def test_tokyo(self, tokyo_dataset, tokyo_table, project_id): assert df["max_year"][0] >= 2000 -@pytest.mark.skip(reason="large query for BQ Storage API tests") -def test_read_gbq_w_bqstorage_api(credentials): +@pytest.mark.slow(reason="Large query for BQ Storage API tests.") +def test_read_gbq_w_bqstorage_api(credentials, random_dataset): + pytest.importorskip("google.cloud.bigquery_storage") df = gbq.read_gbq( """ SELECT - dependency_name, - dependency_platform, - project_name, - project_id, - version_number, - version_id, - dependency_kind, - optional_dependency, - dependency_requirements, - dependency_project_id - FROM - `bigquery-public-data.libraries_io.dependencies` - WHERE - LOWER(dependency_platform) = 'npm' - LIMIT 2500000 + total_amount, + passenger_count, + trip_distance + FROM `bigquery-public-data.new_york_taxi_trips.tlc_green_trips_2014` + -- Select non-null rows for no-copy conversion from Arrow to pandas. + WHERE total_amount IS NOT NULL + AND passenger_count IS NOT NULL + AND trip_distance IS NOT NULL + LIMIT 10000000 """, use_bqstorage_api=True, credentials=credentials, + configuration={ + "query": { + "destinationTable": { + "projectId": random_dataset.project, + "datasetId": random_dataset.dataset_id, + "tableId": "".join( + [ + "test_read_gbq_w_bqstorage_api_", + str(uuid.uuid4()).replace("-", "_"), + ] + ), + }, + "writeDisposition": "WRITE_TRUNCATE", + } + }, ) - assert len(df) == 2500000 + assert len(df) == 10000000 class TestToGBQIntegration(object): diff --git a/packages/pandas-gbq/tests/unit/conftest.py b/packages/pandas-gbq/tests/unit/conftest.py index c1d9e0f8a362..132b37baf30b 100644 --- a/packages/pandas-gbq/tests/unit/conftest.py +++ b/packages/pandas-gbq/tests/unit/conftest.py @@ -15,12 +15,14 @@ def reset_context(): @pytest.fixture(autouse=True) def mock_bigquery_client(monkeypatch): - from pandas_gbq import gbq from google.api_core.exceptions import NotFound import google.cloud.bigquery import google.cloud.bigquery.table mock_client = mock.create_autospec(google.cloud.bigquery.Client) + # Constructor returns the mock itself, so this mock can be treated as the + # constructor or the instance. + mock_client.return_value = mock_client mock_schema = [google.cloud.bigquery.SchemaField("_f0", "INTEGER")] # Mock out SELECT 1 query results. mock_query = mock.create_autospec(google.cloud.bigquery.QueryJob) @@ -34,5 +36,6 @@ def mock_bigquery_client(monkeypatch): mock_client.query.return_value = mock_query # Mock table creation. mock_client.get_table.side_effect = NotFound("nope") - monkeypatch.setattr(gbq.GbqConnector, "get_client", lambda _: mock_client) + monkeypatch.setattr(google.cloud.bigquery, "Client", mock_client) + mock_client.reset_mock() return mock_client diff --git a/packages/pandas-gbq/tests/unit/test_auth.py b/packages/pandas-gbq/tests/unit/test_auth.py index fa054fe1efbe..d0a7cbf6cddd 100644 --- a/packages/pandas-gbq/tests/unit/test_auth.py +++ b/packages/pandas-gbq/tests/unit/test_auth.py @@ -72,8 +72,6 @@ def mock_default_credentials(scopes=None, request=None): ) monkeypatch.setattr(google.auth, "default", mock_default_credentials) - mock_client = mock.create_autospec(google.cloud.bigquery.Client) - monkeypatch.setattr(google.cloud.bigquery, "Client", mock_client) credentials, project = auth.get_credentials() assert project == "default-project" diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index 0452bf8882b4..ddb4e5f9db92 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -5,6 +5,7 @@ from unittest import mock import numpy +import pandas from pandas import DataFrame import pandas.util.testing as tm import pkg_resources @@ -22,6 +23,10 @@ ).parsed_version +def _make_connector(project_id="some-project", **kwargs): + return gbq.GbqConnector(project_id, **kwargs) + + @pytest.fixture def min_bq_version(): import pkg_resources @@ -99,7 +104,7 @@ def no_auth(monkeypatch): ("DATETIME", "datetime64[ns]"), ], ) -def test_should_return_bigquery_correctly_typed(type_, expected): +def test__bqschema_to_nullsafe_dtypes(type_, expected): result = gbq._bqschema_to_nullsafe_dtypes( [dict(name="x", type=type_, mode="NULLABLE")] ) @@ -109,6 +114,35 @@ def test_should_return_bigquery_correctly_typed(type_, expected): assert result == {"x": expected} +def test_GbqConnector_get_client_w_old_bq(monkeypatch, mock_bigquery_client): + gbq._test_google_api_imports() + connector = _make_connector() + monkeypatch.setattr(gbq, "HAS_CLIENT_INFO", False) + + connector.get_client() + + # No client_info argument. + mock_bigquery_client.assert_called_with( + credentials=mock.ANY, project=mock.ANY + ) + + +def test_GbqConnector_get_client_w_new_bq(mock_bigquery_client): + gbq._test_google_api_imports() + pytest.importorskip( + "google.cloud.bigquery", minversion=gbq.BIGQUERY_CLIENT_INFO_VERSION + ) + pytest.importorskip("google.api_core.client_info") + + connector = _make_connector() + connector.get_client() + + _, kwargs = mock_bigquery_client.call_args + assert kwargs["client_info"].user_agent == "pandas-{}".format( + pandas.__version__ + ) + + def test_to_gbq_should_fail_if_invalid_table_name_passed(): with pytest.raises(gbq.NotFoundException): gbq.to_gbq(DataFrame([[1]]), "invalid_table_name", project_id="1234") From 7db56eaf7d21aeabbab40c64ed26db4e2c902898 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 29 Jul 2019 11:15:40 -0700 Subject: [PATCH 193/519] Release 0.11.0 --- packages/pandas-gbq/docs/source/changelog.rst | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 5e77de5d600a..75f57e6e54e9 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -3,13 +3,18 @@ Changelog .. _changelog-0.11.0: -0.11.0 / TBD ------------- +0.11.0 / 2019-07-29 +------------------- - **Breaking Change:** Python 2 support has been dropped. This is to align with the pandas package which dropped Python 2 support at the end of 2019. (:issue:`268`) +Enhancements +~~~~~~~~~~~~ + +- Ensure ``table_schema`` argument is not modified inplace. (:issue:`278`) + Implementation changes ~~~~~~~~~~~~~~~~~~~~~~ @@ -20,6 +25,9 @@ Internal changes ~~~~~~~~~~~~~~~~ - Populate ``user-agent`` with ``pandas`` version information. (:issue:`281`) +- Fix ``pytest.raises`` usage for latest pytest. Fix warnings in tests. + (:issue:`282`) +- Update CI to install nightly packages in the conda tests. (:issue:`254`) .. _changelog-0.10.0: From 5634c9df50e58b714039a2e945a72c6a04275d67 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 9 Aug 2019 15:30:29 -0700 Subject: [PATCH 194/519] ENH: Add max_results argument to read_gbq. (#286) * ENH: Add max_results argument to read_gbq. This argument allows you to set a maximum number of rows in the resulting dataframe. Set to 0 to ignore any results. * Call get_table on destination so listing works on older versions of the client library. * Add max_results to changelog. * Fix unit tests for extra call to get_table. * Add unit tests for max_results. * Adjust fail-under percent --- packages/pandas-gbq/docs/source/changelog.rst | 10 +++++ packages/pandas-gbq/noxfile.py | 2 +- packages/pandas-gbq/pandas_gbq/gbq.py | 38 ++++++++++++++++--- packages/pandas-gbq/tests/system/test_gbq.py | 24 ++++++++++++ packages/pandas-gbq/tests/unit/conftest.py | 2 - packages/pandas-gbq/tests/unit/test_gbq.py | 33 ++++++++-------- 6 files changed, 85 insertions(+), 24 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 75f57e6e54e9..6ba5295ef331 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,6 +1,16 @@ Changelog ========= +.. _changelog-0.12.0: + +0.12.0 / TBD +------------ + +- Add ``max_results`` argument to :func:`~pandas_gbq.read_gbq()`. Use this + argument to limit the number of rows in the results DataFrame. Set + ``max_results`` to 0 to ignore query outputs, such as for DML or DDL + queries. (:issue:`102`) + .. _changelog-0.11.0: 0.11.0 / 2019-07-29 diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index d13a732b4d33..13c38ce19a7e 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -48,7 +48,7 @@ def unit(session): @nox.session def cover(session, python=latest_python): session.install("coverage", "pytest-cov") - session.run("coverage", "report", "--show-missing", "--fail-under=40") + session.run("coverage", "report", "--show-missing", "--fail-under=74") session.run("coverage", "erase") diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 507432db0674..a332a6c2bf50 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -436,7 +436,7 @@ def process_http_error(ex): raise GenericGBQException("Reason: {0}".format(ex)) - def run_query(self, query, **kwargs): + def run_query(self, query, max_results=None, **kwargs): from concurrent.futures import TimeoutError from google.auth.exceptions import RefreshError from google.cloud import bigquery @@ -526,15 +526,33 @@ def run_query(self, query, **kwargs): ) ) + return self._download_results(query_reply, max_results=max_results) + + def _download_results(self, query_job, max_results=None): + # No results are desired, so don't bother downloading anything. + if max_results == 0: + return None + + if max_results is None: + # Only use the BigQuery Storage API if the full result set is requested. + bqstorage_client = self.bqstorage_client + else: + bqstorage_client = None + try: - rows_iter = query_reply.result() + query_job.result() + # Get the table schema, so that we can list rows. + destination = self.client.get_table(query_job.destination) + rows_iter = self.client.list_rows( + destination, max_results=max_results + ) except self.http_error as ex: self.process_http_error(ex) schema_fields = [field.to_api_repr() for field in rows_iter.schema] nullsafe_dtypes = _bqschema_to_nullsafe_dtypes(schema_fields) df = rows_iter.to_dataframe( - dtypes=nullsafe_dtypes, bqstorage_client=self.bqstorage_client + dtypes=nullsafe_dtypes, bqstorage_client=bqstorage_client ) if df.empty: @@ -812,6 +830,7 @@ def read_gbq( configuration=None, credentials=None, use_bqstorage_api=False, + max_results=None, verbose=None, private_key=None, ): @@ -907,9 +926,16 @@ def read_gbq( ``configuration`` dictionary. This feature requires the ``google-cloud-bigquery-storage`` and - ``fastavro`` packages. + ``pyarrow`` packages. + + This value is ignored if ``max_results`` is set. .. versionadded:: 0.10.0 + max_results : int, optional + If set, limit the maximum number of rows to fetch from the query + results. + + .. versionadded:: 0.12.0 verbose : None, deprecated Deprecated in Pandas-GBQ 0.4.0. Use the `logging module to adjust verbosity instead @@ -969,7 +995,9 @@ def read_gbq( use_bqstorage_api=use_bqstorage_api, ) - final_df = connector.run_query(query, configuration=configuration) + final_df = connector.run_query( + query, configuration=configuration, max_results=max_results + ) # Reindex the DataFrame on the provided column if index_col is not None: diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index ebe7782c3138..5b89248cde65 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -584,6 +584,30 @@ def test_download_dataset_larger_than_200k_rows(self, project_id): ) assert len(df.drop_duplicates()) == test_size + def test_ddl(self, random_dataset, project_id): + # Bug fix for https://github.com/pydata/pandas-gbq/issues/45 + df = gbq.read_gbq( + "CREATE OR REPLACE TABLE {}.test_ddl (x INT64)".format( + random_dataset.dataset_id + ) + ) + assert len(df) == 0 + + def test_ddl_w_max_results(self, random_dataset, project_id): + df = gbq.read_gbq( + "CREATE OR REPLACE TABLE {}.test_ddl (x INT64)".format( + random_dataset.dataset_id + ), + max_results=0, + ) + assert df is None + + def test_max_results(self, random_dataset, project_id): + df = gbq.read_gbq( + "SELECT * FROM UNNEST(GENERATE_ARRAY(1, 100))", max_results=10 + ) + assert len(df) == 10 + def test_zero_rows(self, project_id): # Bug fix for https://github.com/pandas-dev/pandas/issues/10273 df = gbq.read_gbq( diff --git a/packages/pandas-gbq/tests/unit/conftest.py b/packages/pandas-gbq/tests/unit/conftest.py index 132b37baf30b..490d5663f18b 100644 --- a/packages/pandas-gbq/tests/unit/conftest.py +++ b/packages/pandas-gbq/tests/unit/conftest.py @@ -15,7 +15,6 @@ def reset_context(): @pytest.fixture(autouse=True) def mock_bigquery_client(monkeypatch): - from google.api_core.exceptions import NotFound import google.cloud.bigquery import google.cloud.bigquery.table @@ -35,7 +34,6 @@ def mock_bigquery_client(monkeypatch): mock_query.result.return_value = mock_rows mock_client.query.return_value = mock_query # Mock table creation. - mock_client.get_table.side_effect = NotFound("nope") monkeypatch.setattr(google.cloud.bigquery, "Client", mock_client) mock_client.reset_mock() return mock_client diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index ddb4e5f9db92..388e95e888b9 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -34,10 +34,6 @@ def min_bq_version(): return pkg_resources.parse_version("1.9.0") -def mock_none_credentials(*args, **kwargs): - return None, None - - def mock_get_credentials_no_project(*args, **kwargs): import google.auth.credentials @@ -76,16 +72,6 @@ def mock_compute_engine_credentials(): return mock_credentials -@pytest.fixture -def mock_get_user_credentials(*args, **kwargs): - import google.auth.credentials - - mock_credentials = mock.create_autospec( - google.auth.credentials.Credentials - ) - return mock_credentials - - @pytest.fixture(autouse=True) def no_auth(monkeypatch): import pydata_google_auth @@ -351,6 +337,17 @@ def test_read_gbq_without_inferred_project_id_from_compute_engine_credentials( ) +def test_read_gbq_with_max_results_zero(monkeypatch): + df = gbq.read_gbq("SELECT 1", dialect="standard", max_results=0) + assert df is None + + +def test_read_gbq_with_max_results_ten(monkeypatch, mock_bigquery_client): + df = gbq.read_gbq("SELECT 1", dialect="standard", max_results=10) + assert df is not None + mock_bigquery_client.list_rows.assert_called_with(mock.ANY, max_results=10) + + def test_read_gbq_with_invalid_private_key_json_should_fail(): with pytest.raises(pandas_gbq.exceptions.InvalidPrivateKeyFormat): gbq.read_gbq( @@ -511,8 +508,12 @@ def test_generate_bq_schema_deprecated(): gbq.generate_bq_schema(df) -def test_load_does_not_modify_schema_arg(): - # Test of Issue # 277 +def test_load_does_not_modify_schema_arg(mock_bigquery_client): + """Test of Issue # 277.""" + from google.api_core.exceptions import NotFound + + # Create table with new schema. + mock_bigquery_client.get_table.side_effect = NotFound("nope") df = DataFrame( { "field1": ["a", "b"], From 4ef93886b696e8abcd9790019b373d98bf7e1c37 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 13 Aug 2019 10:24:05 -0700 Subject: [PATCH 195/519] DOC: Add code samples to introduction and refactor howto guides. (#287) * DOC: Add code samples to introduction and refactor howto guides. This change adds a proper introduction article to introduce `read_gbq` and `to_gbq` with code samples. To reduce the amount of duplicated content, code samples have been extracted into sample files. To give a better next steps experience, the content in the authentication, reading, and writing guides has been rearranged to have the most relevant next steps (after the introduction) as the first section. * Move private_key_contents fixture to conftest * Autouse credentials, so that samples tests are skipped when credentials are not available. * Run system tests on Circle CI by checking for a file at `ci/service_account.json`. --- packages/pandas-gbq/conftest.py | 73 ++++++++++++ packages/pandas-gbq/docs/source/changelog.rst | 10 ++ .../docs/source/howto/authentication.rst | 111 +++++++++--------- packages/pandas-gbq/docs/source/index.rst | 2 +- packages/pandas-gbq/docs/source/install.rst | 7 +- packages/pandas-gbq/docs/source/intro.rst | 59 ++++++++-- packages/pandas-gbq/docs/source/reading.rst | 99 ++++++++-------- packages/pandas-gbq/docs/source/samples | 1 + packages/pandas-gbq/docs/source/writing.rst | 31 ++--- packages/pandas-gbq/noxfile.py | 6 +- packages/pandas-gbq/samples/__init__.py | 0 .../pandas-gbq/samples/read_gbq_legacy.py | 35 ++++++ .../pandas-gbq/samples/read_gbq_simple.py | 32 +++++ packages/pandas-gbq/samples/samples | 1 + packages/pandas-gbq/samples/tests/__init__.py | 0 packages/pandas-gbq/samples/tests/conftest.py | 13 ++ .../pandas-gbq/samples/tests/test_read_gbq.py | 22 ++++ .../pandas-gbq/samples/tests/test_to_gbq.py | 14 +++ packages/pandas-gbq/samples/to_gbq_simple.py | 42 +++++++ packages/pandas-gbq/tests/system/conftest.py | 46 -------- packages/pandas-gbq/tests/system/test_gbq.py | 30 ----- 21 files changed, 421 insertions(+), 213 deletions(-) create mode 100644 packages/pandas-gbq/conftest.py create mode 120000 packages/pandas-gbq/docs/source/samples create mode 100644 packages/pandas-gbq/samples/__init__.py create mode 100644 packages/pandas-gbq/samples/read_gbq_legacy.py create mode 100644 packages/pandas-gbq/samples/read_gbq_simple.py create mode 120000 packages/pandas-gbq/samples/samples create mode 100644 packages/pandas-gbq/samples/tests/__init__.py create mode 100644 packages/pandas-gbq/samples/tests/conftest.py create mode 100644 packages/pandas-gbq/samples/tests/test_read_gbq.py create mode 100644 packages/pandas-gbq/samples/tests/test_to_gbq.py create mode 100644 packages/pandas-gbq/samples/to_gbq_simple.py delete mode 100644 packages/pandas-gbq/tests/system/conftest.py diff --git a/packages/pandas-gbq/conftest.py b/packages/pandas-gbq/conftest.py new file mode 100644 index 000000000000..1485e41eb4b9 --- /dev/null +++ b/packages/pandas-gbq/conftest.py @@ -0,0 +1,73 @@ +"""Shared pytest fixtures for system tests.""" + +import os +import os.path +import uuid + +import google.oauth2.service_account +import pytest + + +@pytest.fixture(scope="session") +def project_id(): + return os.environ.get("GBQ_PROJECT_ID") or os.environ.get( + "GOOGLE_CLOUD_PROJECT" + ) # noqa + + +@pytest.fixture(scope="session") +def private_key_path(): + path = os.path.join( + "ci", "service_account.json" + ) # Written by the 'ci/config_auth.sh' script. + if "GBQ_GOOGLE_APPLICATION_CREDENTIALS" in os.environ: + path = os.environ["GBQ_GOOGLE_APPLICATION_CREDENTIALS"] + elif "GOOGLE_APPLICATION_CREDENTIALS" in os.environ: + path = os.environ["GOOGLE_APPLICATION_CREDENTIALS"] + + if not os.path.isfile(path): + pytest.skip( + "Cannot run integration tests when there is " + "no file at the private key json file path" + ) + return None + + return path + + +@pytest.fixture(scope="session") +def private_key_contents(private_key_path): + if private_key_path is None: + return None + + with open(private_key_path) as f: + return f.read() + + +@pytest.fixture(scope="module") +def bigquery_client(project_id, private_key_path): + from google.cloud import bigquery + + return bigquery.Client.from_service_account_json( + private_key_path, project=project_id + ) + + +@pytest.fixture() +def random_dataset_id(bigquery_client): + import google.api_core.exceptions + + dataset_id = "".join(["pandas_gbq_", str(uuid.uuid4()).replace("-", "_")]) + dataset_ref = bigquery_client.dataset(dataset_id) + yield dataset_id + try: + bigquery_client.delete_dataset(dataset_ref, delete_contents=True) + except google.api_core.exceptions.NotFound: + pass # Not all tests actually create a dataset + + +@pytest.fixture() +def credentials(private_key_path): + return google.oauth2.service_account.Credentials.from_service_account_file( + private_key_path + ) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 6ba5295ef331..3a71ebc51917 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -11,6 +11,12 @@ Changelog ``max_results`` to 0 to ignore query outputs, such as for DML or DDL queries. (:issue:`102`) +Documentation +~~~~~~~~~~~~~ + +- Add code samples to introduction and refactor howto guides. (:issue:`239`) + + .. _changelog-0.11.0: 0.11.0 / 2019-07-29 @@ -44,6 +50,10 @@ Internal changes 0.10.0 / 2019-04-05 ------------------- +- **Breaking Change:** Default SQL dialect is now ``standard``. Use + :attr:`pandas_gbq.context.dialect` to override the default value. + (:issue:`195`, :issue:`245`) + Documentation ~~~~~~~~~~~~~ diff --git a/packages/pandas-gbq/docs/source/howto/authentication.rst b/packages/pandas-gbq/docs/source/howto/authentication.rst index 20b0674a2bdc..70ec143f3f54 100644 --- a/packages/pandas-gbq/docs/source/howto/authentication.rst +++ b/packages/pandas-gbq/docs/source/howto/authentication.rst @@ -1,11 +1,68 @@ Authentication ============== +Before you begin, you must create a Google Cloud Platform project. Use the +`BigQuery sandbox `__ to try +the service for free. + pandas-gbq `authenticates with the Google BigQuery service -`_ via OAuth 2.0. +`_ via OAuth 2.0. Use +the ``credentials`` argument to explicitly pass in Google +:class:`~google.auth.credentials.Credentials`. .. _authentication: +Default Authentication Methods +------------------------------ + +If the ``credentials`` parameter is not set, pandas-gbq tries the following +authentication methods: + +1. In-memory, cached credentials at ``pandas_gbq.context.credentials``. See + :attr:`pandas_gbq.Context.credentials` for details. + + .. code:: python + + import pandas_gbq + + credentials = ... # From google-auth or pydata-google-auth library. + + # Update the in-memory credentials cache (added in pandas-gbq 0.7.0). + pandas_gbq.context.credentials = credentials + pandas_gbq.context.project = "your-project-id" + + # The credentials and project_id arguments can be omitted. + df = pandas_gbq.read_gbq("SELECT my_col FROM `my_dataset.my_table`") + +2. Application Default Credentials via the :func:`google.auth.default` + function. + + .. note:: + + If pandas-gbq can obtain default credentials but those credentials + cannot be used to query BigQuery, pandas-gbq will also try obtaining + user account credentials. + + A common problem with default credentials when running on Google + Compute Engine is that the VM does not have sufficient scopes to query + BigQuery. + +3. User account credentials. + + pandas-gbq loads cached credentials from a hidden user folder on the + operating system. + + Windows + ``%APPDATA%\pandas_gbq\bigquery_credentials.dat`` + + Linux/Mac/Unix + ``~/.config/pandas_gbq/bigquery_credentials.dat`` + + If pandas-gbq does not find cached credentials, it prompts you to open a + web browser, where you can grant pandas-gbq permissions to access your + cloud resources. These credentials are only used locally. See the + :doc:`privacy policy ` for details. + Authenticating with a Service Account -------------------------------------- @@ -131,55 +188,3 @@ credentials are not found. Additional information on the user credentials authentication mechanism can be found in the `Google Cloud authentication guide `__. - - -Default Authentication Methods ------------------------------- - -If the ``credentials`` parameter (or the deprecated ``private_key`` -parameter) is ``None``, pandas-gbq tries the following authentication -methods: - -1. In-memory, cached credentials at ``pandas_gbq.context.credentials``. See - :attr:`pandas_gbq.Context.credentials` for details. - - .. code:: python - - import pandas_gbq - - credentials = ... # From google-auth or pydata-google-auth library. - - # Update the in-memory credentials cache (added in pandas-gbq 0.7.0). - pandas_gbq.context.credentials = credentials - pandas_gbq.context.project = "your-project-id" - - # The credentials and project_id arguments can be omitted. - df = pandas_gbq.read_gbq("SELECT my_col FROM `my_dataset.my_table`") - -2. Application Default Credentials via the :func:`google.auth.default` - function. - - .. note:: - - If pandas-gbq can obtain default credentials but those credentials - cannot be used to query BigQuery, pandas-gbq will also try obtaining - user account credentials. - - A common problem with default credentials when running on Google - Compute Engine is that the VM does not have sufficient scopes to query - BigQuery. - -3. User account credentials. - - pandas-gbq loads cached credentials from a hidden user folder on the - operating system. - - Windows - ``%APPDATA%\pandas_gbq\bigquery_credentials.dat`` - - Linux/Mac/Unix - ``~/.config/pandas_gbq/bigquery_credentials.dat`` - - If pandas-gbq does not find cached credentials, it opens a browser window - asking for you to authenticate to your BigQuery account using the product - name ``pandas GBQ``. diff --git a/packages/pandas-gbq/docs/source/index.rst b/packages/pandas-gbq/docs/source/index.rst index caadd7954a62..bfb51d9eadda 100644 --- a/packages/pandas-gbq/docs/source/index.rst +++ b/packages/pandas-gbq/docs/source/index.rst @@ -13,7 +13,7 @@ with a shape and data types derived from the source table. Additionally, DataFrames can be inserted into new BigQuery tables or appended to existing tables. -.. warning:: +.. note:: To use this module, you will need a valid BigQuery account. Use the `BigQuery sandbox `__ to diff --git a/packages/pandas-gbq/docs/source/install.rst b/packages/pandas-gbq/docs/source/install.rst index 457a33e9ae85..6810a44a51e9 100644 --- a/packages/pandas-gbq/docs/source/install.rst +++ b/packages/pandas-gbq/docs/source/install.rst @@ -40,15 +40,16 @@ This module requires following additional dependencies: - `pydata-google-auth `__: Helpers for authentication to Google's API - `google-auth `__: authentication and authorization for Google's API - `google-auth-oauthlib `__: integration with `oauthlib `__ for end-user authentication -- `google-cloud-bigquery `__: Google Cloud client library for BigQuery +- `google-cloud-bigquery `__: Google Cloud client library for BigQuery +- `google-cloud-bigquery-storage `__: Google Cloud client library for BigQuery Storage API .. note:: - The dependency on `google-cloud-bigquery `__ is new in version 0.3.0 of ``pandas-gbq``. + The dependency on `google-cloud-bigquery `__ is new in version 0.3.0 of ``pandas-gbq``. Versions less than 0.3.0 required the following dependencies: - `httplib2 `__: HTTP client (no longer required) - - `google-api-python-client `__: Google's API client (no longer required, replaced by `google-cloud-bigquery `__:) + - `google-api-python-client `__: Google's API client (no longer required, replaced by `google-cloud-bigquery `__:) - `google-auth `__: authentication and authorization for Google's API - `google-auth-oauthlib `__: integration with `oauthlib `__ for end-user authentication - `google-auth-httplib2 `__: adapter to use ``httplib2`` HTTP client with ``google-auth`` (no longer required) diff --git a/packages/pandas-gbq/docs/source/intro.rst b/packages/pandas-gbq/docs/source/intro.rst index 0e1a6a3b0374..e506ebe78b56 100644 --- a/packages/pandas-gbq/docs/source/intro.rst +++ b/packages/pandas-gbq/docs/source/intro.rst @@ -1,16 +1,43 @@ Introduction ============ -Supported Data Types -++++++++++++++++++++ +The pandas-gbq package reads data from `Google BigQuery +`__ to a :class:`pandas.DataFrame` +object and also writes :class:`pandas.DataFrame` objects to BigQuery tables. -Pandas supports all these `BigQuery data types `__: -``STRING``, ``INTEGER`` (64bit), ``FLOAT`` (64 bit), ``BOOLEAN`` and -``TIMESTAMP`` (microsecond precision). Data types ``BYTES`` and ``RECORD`` -are not supported. +Authenticating to BigQuery +-------------------------- -Logging -+++++++ +Before you begin, you must create a Google Cloud Platform project. Use the +`BigQuery sandbox `__ to try +the service for free. + +If you do not provide any credentials, this module attempts to load +credentials from the environment. If no credentials are found, pandas-gbq +prompts you to open a web browser, where you can grant it permissions to +access your cloud resources. These credentials are only used locally. See the +:doc:`privacy policy ` for details. + +Learn about authentication methods in the :doc:`authentication guide +`. + +Reading data from BigQuery +-------------------------- + +Use the :func:`pandas_gbq.read_gbq` function to run a BigQuery query and +download the results as a :class:`pandas.DataFrame` object. + +.. literalinclude:: samples/read_gbq_simple.py + :language: python + :dedent: 4 + :start-after: [START bigquery_pandas_gbq_read_gbq_simple] + :end-before: [END bigquery_pandas_gbq_read_gbq_simple] + +By default, queries use standard SQL syntax. Visit the :doc:`reading tables +guide ` to learn about the available options. + +Adjusting log vebosity +^^^^^^^^^^^^^^^^^^^^^^ Because some requests take some time, this library will log its progress of longer queries. IPython & Jupyter by default attach a handler to the logger. @@ -23,3 +50,19 @@ more verbose logs, you can do something like: logger = logging.getLogger('pandas_gbq') logger.setLevel(logging.DEBUG) logger.addHandler(logging.StreamHandler()) + +Writing data to BigQuery +------------------------ + +Use the :func:`pandas_gbq.to_gbq` function to write a +:class:`pandas.DataFrame` object to a BigQuery table. + +.. literalinclude:: samples/to_gbq_simple.py + :language: python + :dedent: 4 + :start-after: [START bigquery_pandas_gbq_to_gbq_simple] + :end-before: [END bigquery_pandas_gbq_to_gbq_simple] + +The destination table and destination dataset will automatically be created. +By default, writes to BigQuery fail if the table already exists. Visit the +:doc:`writing tables guide ` to learn about the available options. diff --git a/packages/pandas-gbq/docs/source/reading.rst b/packages/pandas-gbq/docs/source/reading.rst index 0be39b14e6c4..67919c57f253 100644 --- a/packages/pandas-gbq/docs/source/reading.rst +++ b/packages/pandas-gbq/docs/source/reading.rst @@ -3,26 +3,20 @@ Reading Tables ============== -Suppose you want to load all data from an existing BigQuery table -``test_dataset.test_table`` into a DataFrame using the -:func:`~pandas_gbq.read_gbq` function. +Use the :func:`pandas_gbq.read_gbq` function to run a BigQuery query and +download the results as a :class:`pandas.DataFrame` object. -.. code-block:: python - - import pandas_gbq - - # TODO: Set your BigQuery Project ID. - projectid = "xxxxxxxx" - - data_frame = pandas_gbq.read_gbq( - 'SELECT * FROM `test_dataset.test_table`', - project_id=projectid) +.. literalinclude:: samples/read_gbq_simple.py + :language: python + :dedent: 4 + :start-after: [START bigquery_pandas_gbq_read_gbq_simple] + :end-before: [END bigquery_pandas_gbq_read_gbq_simple] .. note:: - A project ID is sometimes optional if it can be inferred during - authentication, but it is required when authenticating with user - credentials. You can find your project ID in the `Google Cloud console + A project ID is optional if it can be inferred during authentication, but + it is required when authenticating with user credentials. You can find + your project ID in the `Google Cloud console `__. You can define which column from BigQuery to use as an index in the @@ -36,44 +30,31 @@ destination DataFrame as well as a preferred column order as follows: index_col='index_column_name', col_order=['col1', 'col2', 'col3']) - -You can specify the query config as parameter to use additional options of -your job. For more information about query configuration parameters see `here -`__. - -.. code-block:: python - - configuration = { - 'query': { - "useQueryCache": False - } - } - data_frame = read_gbq( - 'SELECT * FROM `test_dataset.test_table`', - project_id=projectid, - configuration=configuration) - +Querying with legacy SQL syntax +------------------------------- The ``dialect`` argument can be used to indicate whether to use -BigQuery's ``'legacy'`` SQL or BigQuery's ``'standard'`` SQL (beta). The -default value is ``'standard'`` For more information on BigQuery's standard -SQL, see `BigQuery SQL Reference -`__ - -.. code-block:: python +BigQuery's ``'legacy'`` SQL or BigQuery's ``'standard'`` SQL. The +default value is ``'standard'``. - data_frame = pandas_gbq.read_gbq( - 'SELECT * FROM [test_dataset.test_table]', - project_id=projectid, - dialect='legacy') +.. literalinclude:: samples/read_gbq_legacy.py + :language: python + :dedent: 4 + :start-after: [START bigquery_pandas_gbq_read_gbq_legacy] + :end-before: [END bigquery_pandas_gbq_read_gbq_legacy] +* `Standard SQL reference + `__ +* `Legacy SQL reference + `__ .. _reading-dtypes: Inferring the DataFrame's dtypes -------------------------------- -The :func:`~pandas_gbq.read_gbq` method infers the pandas dtype for each column, based on the BigQuery table schema. +The :func:`~pandas_gbq.read_gbq` method infers the pandas dtype for each +column, based on the BigQuery table schema. ================== ========================= BigQuery Data Type dtype @@ -87,7 +68,7 @@ DATE datetime64[ns] .. _reading-bqstorage-api: -Using the BigQuery Storage API +Improving download performance ------------------------------ Use the BigQuery Storage API to download large (>125 MB) query results more @@ -105,22 +86,42 @@ quickly (but at an `increased cost create BigQuery Storage API read sessions. This permission is provided by the `bigquery.user role `__. -#. Install the ``google-cloud-bigquery-storage``, ``fastavro``, and - ``python-snappy`` packages. +#. Install the ``google-cloud-bigquery-storage`` and ``pyarrow`` + packages. With pip: .. code-block:: sh - pip install --upgrade google-cloud-bigquery-storage fastavro python-snappy + pip install --upgrade google-cloud-bigquery-storage pyarrow With conda: .. code-block:: sh - conda install -c conda-forge google-cloud-bigquery-storage fastavro python-snappy + conda install -c conda-forge google-cloud-bigquery-storage #. Set ``use_bqstorage_api`` to ``True`` when calling the :func:`~pandas_gbq.read_gbq` function. As of the ``google-cloud-bigquery`` package, version 1.11.1 or later,the function will fallback to the BigQuery API if the BigQuery Storage API cannot be used, such as with small query results. + +Advanced configuration +---------------------- + +You can specify the query config as parameter to use additional options of +your job. Refer to the `JobConfiguration REST resource reference +`__ +for details. + +.. code-block:: python + + configuration = { + 'query': { + "useQueryCache": False + } + } + data_frame = read_gbq( + 'SELECT * FROM `test_dataset.test_table`', + project_id=projectid, + configuration=configuration) diff --git a/packages/pandas-gbq/docs/source/samples b/packages/pandas-gbq/docs/source/samples new file mode 120000 index 000000000000..47920198b9bd --- /dev/null +++ b/packages/pandas-gbq/docs/source/samples @@ -0,0 +1 @@ +../../samples \ No newline at end of file diff --git a/packages/pandas-gbq/docs/source/writing.rst b/packages/pandas-gbq/docs/source/writing.rst index 0c5b41e0cc7e..a6a5c1238320 100644 --- a/packages/pandas-gbq/docs/source/writing.rst +++ b/packages/pandas-gbq/docs/source/writing.rst @@ -1,33 +1,20 @@ .. _writer: -Writing DataFrames -================== +Writing Tables +============== -Assume we want to write a :class:`~pandas.DataFrame` named ``df`` into a -BigQuery table using :func:`~pandas_gbq.to_gbq`. +Use the :func:`pandas_gbq.to_gbq` function to write a +:class:`pandas.DataFrame` object to a BigQuery table. -.. ipython:: python - - import pandas as pd - df = pd.DataFrame({'my_string': list('abc'), - 'my_int64': list(range(1, 4)), - 'my_float64': np.arange(4.0, 7.0), - 'my_bool1': [True, False, True], - 'my_bool2': [False, True, False], - 'my_dates': pd.date_range('now', periods=3)}) - - df - df.dtypes - -.. code-block:: python - - import pandas_gbq - pandas_gbq.to_gbq(df, 'my_dataset.my_table', project_id=projectid) +.. literalinclude:: samples/to_gbq_simple.py + :language: python + :dedent: 4 + :start-after: [START bigquery_pandas_gbq_to_gbq_simple] + :end-before: [END bigquery_pandas_gbq_to_gbq_simple] The destination table and destination dataset will automatically be created if they do not already exist. - Writing to an Existing Table ---------------------------- diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 13c38ce19a7e..f0eadfd109df 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -95,5 +95,9 @@ def system(session): additional_args = additional_args + ["-m", "not local_auth"] session.run( - "pytest", os.path.join(".", "tests", "system"), "-v", *additional_args + "pytest", + os.path.join(".", "tests", "system"), + os.path.join(".", "samples", "tests"), + "-v", + *additional_args ) diff --git a/packages/pandas-gbq/samples/__init__.py b/packages/pandas-gbq/samples/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/packages/pandas-gbq/samples/read_gbq_legacy.py b/packages/pandas-gbq/samples/read_gbq_legacy.py new file mode 100644 index 000000000000..ed2e4f408f27 --- /dev/null +++ b/packages/pandas-gbq/samples/read_gbq_legacy.py @@ -0,0 +1,35 @@ +# Copyright 2019. PyData Development Team +# Distributed under BSD 3-Clause License. +# See LICENSE.txt for details. + +"""Simple query example.""" + +import argparse + +import pandas_gbq + + +def main(project_id): + # [START bigquery_pandas_gbq_read_gbq_legacy] + sql = """ + SELECT country_name, alpha_2_code + FROM [bigquery-public-data:utility_us.country_code_iso] + WHERE alpha_2_code LIKE 'Z%' + """ + df = pandas_gbq.read_gbq( + sql, + project_id=project_id, + # Set the dialect to "legacy" to use legacy SQL syntax. As of + # pandas-gbq version 0.10.0, the default dialect is "standard". + dialect="legacy", + ) + # [END bigquery_pandas_gbq_read_gbq_legacy] + print(df) + return df + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("project_id") + args = parser.parse_args() + main(args.project_id) diff --git a/packages/pandas-gbq/samples/read_gbq_simple.py b/packages/pandas-gbq/samples/read_gbq_simple.py new file mode 100644 index 000000000000..a7426745bd3f --- /dev/null +++ b/packages/pandas-gbq/samples/read_gbq_simple.py @@ -0,0 +1,32 @@ +# Copyright 2019. PyData Development Team +# Distributed under BSD 3-Clause License. +# See LICENSE.txt for details. + +"""Simple query example.""" + +import argparse + + +def main(project_id): + # [START bigquery_pandas_gbq_read_gbq_simple] + import pandas_gbq + + # TODO: Set project_id to your Google Cloud Platform project ID. + # project_id = "my-project" + + sql = """ + SELECT country_name, alpha_2_code + FROM `bigquery-public-data.utility_us.country_code_iso` + WHERE alpha_2_code LIKE 'A%' + """ + df = pandas_gbq.read_gbq(sql, project_id=project_id) + # [END bigquery_pandas_gbq_read_gbq_simple] + print(df) + return df + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("project_id") + args = parser.parse_args() + main(args.project_id) diff --git a/packages/pandas-gbq/samples/samples b/packages/pandas-gbq/samples/samples new file mode 120000 index 000000000000..d9d97bb8bb7b --- /dev/null +++ b/packages/pandas-gbq/samples/samples @@ -0,0 +1 @@ +docs/source/samples \ No newline at end of file diff --git a/packages/pandas-gbq/samples/tests/__init__.py b/packages/pandas-gbq/samples/tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/packages/pandas-gbq/samples/tests/conftest.py b/packages/pandas-gbq/samples/tests/conftest.py new file mode 100644 index 000000000000..1733fb84842d --- /dev/null +++ b/packages/pandas-gbq/samples/tests/conftest.py @@ -0,0 +1,13 @@ +# Copyright 2019. PyData Development Team +# Distributed under BSD 3-Clause License. +# See LICENSE.txt for details. + +import os + +import pytest + + +@pytest.fixture(autouse=True) +def default_credentials(private_key_path): + """Setup application default credentials for use in code samples.""" + os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = private_key_path diff --git a/packages/pandas-gbq/samples/tests/test_read_gbq.py b/packages/pandas-gbq/samples/tests/test_read_gbq.py new file mode 100644 index 000000000000..1882ded0dd40 --- /dev/null +++ b/packages/pandas-gbq/samples/tests/test_read_gbq.py @@ -0,0 +1,22 @@ +# Copyright 2019. PyData Development Team +# Distributed under BSD 3-Clause License. +# See LICENSE.txt for details. + +"""System tests for read_gbq code samples.""" + +from .. import read_gbq_legacy +from .. import read_gbq_simple + + +def test_read_gbq_legacy(project_id): + df = read_gbq_legacy.main(project_id) + assert df is not None + assert "ZA" in df["alpha_2_code"].values + assert "South Africa" in df["country_name"].values + + +def test_read_gbq_simple(project_id): + df = read_gbq_simple.main(project_id) + assert df is not None + assert "AU" in df["alpha_2_code"].values + assert "Australia" in df["country_name"].values diff --git a/packages/pandas-gbq/samples/tests/test_to_gbq.py b/packages/pandas-gbq/samples/tests/test_to_gbq.py new file mode 100644 index 000000000000..c25b90591cc7 --- /dev/null +++ b/packages/pandas-gbq/samples/tests/test_to_gbq.py @@ -0,0 +1,14 @@ +# Copyright 2019. PyData Development Team +# Distributed under BSD 3-Clause License. +# See LICENSE.txt for details. + +"""System tests for to_gbq code samples.""" + +from .. import to_gbq_simple + + +def test_to_gbq_simple(project_id, bigquery_client, random_dataset_id): + table_id = "{}.to_gbq_simple".format(random_dataset_id) + to_gbq_simple.main(project_id, table_id) + table = bigquery_client.get_table(table_id) + assert table.num_rows == 3 diff --git a/packages/pandas-gbq/samples/to_gbq_simple.py b/packages/pandas-gbq/samples/to_gbq_simple.py new file mode 100644 index 000000000000..37f4cdcd46f2 --- /dev/null +++ b/packages/pandas-gbq/samples/to_gbq_simple.py @@ -0,0 +1,42 @@ +# Copyright 2019. PyData Development Team +# Distributed under BSD 3-Clause License. +# See LICENSE.txt for details. + +"""Simple upload example.""" + +import argparse + + +def main(project_id, table_id): + # [START bigquery_pandas_gbq_to_gbq_simple] + import pandas + import pandas_gbq + + # TODO: Set project_id to your Google Cloud Platform project ID. + # project_id = "my-project" + + # TODO: Set table_id to the full destination table ID (including the + # dataset ID). + # table_id = 'my_dataset.my_table' + + df = pandas.DataFrame( + { + "my_string": ["a", "b", "c"], + "my_int64": [1, 2, 3], + "my_float64": [4.0, 5.0, 6.0], + "my_bool1": [True, False, True], + "my_bool2": [False, True, False], + "my_dates": pandas.date_range("now", periods=3), + } + ) + + pandas_gbq.to_gbq(df, table_id, project_id=project_id) + # [END bigquery_pandas_gbq_to_gbq_simple] + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("project_id") + parser.add_argument("table_id") + args = parser.parse_args() + main(args.project_id, args.table_id) diff --git a/packages/pandas-gbq/tests/system/conftest.py b/packages/pandas-gbq/tests/system/conftest.py deleted file mode 100644 index 5091d71725d5..000000000000 --- a/packages/pandas-gbq/tests/system/conftest.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Shared pytest fixtures for system tests.""" - -import os -import os.path - -import pytest - - -@pytest.fixture(scope="session") -def project_id(): - return os.environ.get("GBQ_PROJECT_ID") or os.environ.get( - "GOOGLE_CLOUD_PROJECT" - ) # noqa - - -@pytest.fixture(scope="session") -def private_key_path(): - path = None - if "GBQ_GOOGLE_APPLICATION_CREDENTIALS" in os.environ: - path = os.environ["GBQ_GOOGLE_APPLICATION_CREDENTIALS"] - elif "GOOGLE_APPLICATION_CREDENTIALS" in os.environ: - path = os.environ["GOOGLE_APPLICATION_CREDENTIALS"] - - if path is None: - pytest.skip( - "Cannot run integration tests without a " - "private key json file path" - ) - return None - if not os.path.isfile(path): - pytest.skip( - "Cannot run integration tests when there is " - "no file at the private key json file path" - ) - return None - - return path - - -@pytest.fixture(scope="session") -def private_key_contents(private_key_path): - if private_key_path is None: - return None - - with open(private_key_path) as f: - return f.read() diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index 5b89248cde65..729cc7ce1f86 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -48,28 +48,6 @@ def gbq_connector(project, credentials): return gbq.GbqConnector(project, credentials=credentials) -@pytest.fixture(scope="module") -def bigquery_client(project_id, private_key_path): - from google.cloud import bigquery - - return bigquery.Client.from_service_account_json( - private_key_path, project=project_id - ) - - -@pytest.fixture() -def random_dataset_id(bigquery_client): - import google.api_core.exceptions - - dataset_id = "".join(["pandas_gbq_", str(uuid.uuid4()).replace("-", "_")]) - dataset_ref = bigquery_client.dataset(dataset_id) - yield dataset_id - try: - bigquery_client.delete_dataset(dataset_ref, delete_contents=True) - except google.api_core.exceptions.NotFound: - pass # Not all tests actually create a dataset - - @pytest.fixture() def random_dataset(bigquery_client, random_dataset_id): from google.cloud import bigquery @@ -150,14 +128,6 @@ def test_should_be_able_to_get_a_bigquery_client(self, gbq_connector): assert bigquery_client is not None -def test_should_read(project, credentials): - query = 'SELECT "PI" AS valid_string' - df = gbq.read_gbq( - query, project_id=project, credentials=credentials, dialect="legacy" - ) - tm.assert_frame_equal(df, DataFrame({"valid_string": ["PI"]})) - - class TestReadGBQIntegration(object): @pytest.fixture(autouse=True) def setup(self, project, credentials): From 498b711e777d9f4fe75eae01b5732207203df431 Mon Sep 17 00:00:00 2001 From: camfrout Date: Sun, 6 Oct 2019 05:52:28 +1100 Subject: [PATCH 196/519] Missing parenthesis in code example (#291) The closing parenthesis is missing in the current code example, making copy/paste of the code failing. --- packages/pandas-gbq/docs/source/howto/authentication.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/docs/source/howto/authentication.rst b/packages/pandas-gbq/docs/source/howto/authentication.rst index 70ec143f3f54..d49632de5e28 100644 --- a/packages/pandas-gbq/docs/source/howto/authentication.rst +++ b/packages/pandas-gbq/docs/source/howto/authentication.rst @@ -160,7 +160,7 @@ credentials are not found. # authorization flow. Note, this doesn't work if you're running from a # notebook on a remote sever, such as over SSH or with Google Colab. auth_local_webserver=True, - + ) df = pandas_gbq.read_gbq( "SELECT my_col FROM `my_dataset.my_table`", From 95c2fffd75935ec9b1b15cb4ce53fb53298f4874 Mon Sep 17 00:00:00 2001 From: Daniel Klevebring Date: Mon, 28 Oct 2019 22:30:15 +0100 Subject: [PATCH 197/519] ENH: show progress bar when downloading data (#292) * add progress bar * update changelog * default to use the progress bar --- packages/pandas-gbq/docs/source/changelog.rst | 3 ++ packages/pandas-gbq/pandas_gbq/gbq.py | 41 ++++++++++++++++--- packages/pandas-gbq/tests/unit/test_gbq.py | 19 +++++++++ 3 files changed, 58 insertions(+), 5 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 3a71ebc51917..8695ef11a961 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -11,6 +11,9 @@ Changelog ``max_results`` to 0 to ignore query outputs, such as for DML or DDL queries. (:issue:`102`) +- Add ``progress_bar_type`` argument to :func:`~pandas_gbq.read_gbq()`. Use this + argument to display a progress bar when downloading data. (:issue:`182`) + Documentation ~~~~~~~~~~~~~ diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index a332a6c2bf50..0da10c4ffd5f 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -436,7 +436,9 @@ def process_http_error(ex): raise GenericGBQException("Reason: {0}".format(ex)) - def run_query(self, query, max_results=None, **kwargs): + def run_query( + self, query, max_results=None, progress_bar_type=None, **kwargs + ): from concurrent.futures import TimeoutError from google.auth.exceptions import RefreshError from google.cloud import bigquery @@ -526,9 +528,15 @@ def run_query(self, query, max_results=None, **kwargs): ) ) - return self._download_results(query_reply, max_results=max_results) + return self._download_results( + query_reply, + max_results=max_results, + progress_bar_type=progress_bar_type, + ) - def _download_results(self, query_job, max_results=None): + def _download_results( + self, query_job, max_results=None, progress_bar_type=None + ): # No results are desired, so don't bother downloading anything. if max_results == 0: return None @@ -552,7 +560,9 @@ def _download_results(self, query_job, max_results=None): schema_fields = [field.to_api_repr() for field in rows_iter.schema] nullsafe_dtypes = _bqschema_to_nullsafe_dtypes(schema_fields) df = rows_iter.to_dataframe( - dtypes=nullsafe_dtypes, bqstorage_client=bqstorage_client + dtypes=nullsafe_dtypes, + bqstorage_client=bqstorage_client, + progress_bar_type=progress_bar_type, ) if df.empty: @@ -833,6 +843,7 @@ def read_gbq( max_results=None, verbose=None, private_key=None, + progress_bar_type="tqdm", ): r"""Load data from Google BigQuery using google-cloud-python @@ -952,6 +963,23 @@ def read_gbq( or string contents. This is useful for remote server authentication (eg. Jupyter/IPython notebook on remote host). + progress_bar_type (Optional[str]): + If set, use the `tqdm `_ library to + display a progress bar while the data downloads. Install the + ``tqdm`` package to use this feature. + Possible values of ``progress_bar_type`` include: + ``None`` + No progress bar. + ``'tqdm'`` + Use the :func:`tqdm.tqdm` function to print a progress bar + to :data:`sys.stderr`. + ``'tqdm_notebook'`` + Use the :func:`tqdm.tqdm_notebook` function to display a + progress bar as a Jupyter notebook widget. + ``'tqdm_gui'`` + Use the :func:`tqdm.tqdm_gui` function to display a + progress bar as a graphical dialog box. + Returns ------- df: DataFrame @@ -996,7 +1024,10 @@ def read_gbq( ) final_df = connector.run_query( - query, configuration=configuration, max_results=max_results + query, + configuration=configuration, + max_results=max_results, + progress_bar_type=progress_bar_type, ) # Reindex the DataFrame on the provided column diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index 388e95e888b9..f213f0169dcf 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -552,3 +552,22 @@ def test_load_does_not_modify_schema_arg(mock_bigquery_client): if_exists="append", ) assert original_schema == original_schema_cp + + +def test_read_gbq_calls_tqdm( + mock_bigquery_client, mock_service_account_credentials +): + mock_service_account_credentials.project_id = "service_account_project_id" + df = gbq.read_gbq( + "SELECT 1", + dialect="standard", + credentials=mock_service_account_credentials, + progress_bar_type="foobar", + ) + assert df is not None + + mock_list_rows = mock_bigquery_client.list_rows("dest", max_results=100) + + mock_list_rows.to_dataframe.assert_called_once_with( + dtypes=mock.ANY, bqstorage_client=mock.ANY, progress_bar_type="foobar" + ) From 0b3161551f71fb5073582c754f8f00dafa93c330 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 25 Nov 2019 13:41:40 -0800 Subject: [PATCH 198/519] FIX: update minimum google-cloud-bigquery version to 1.11.1 (#298) * FIX: update minimum google-cloud-bigquery version to 1.11.1 Version 1.11.0 was the first version to include a progress_bar_type argument. https://googleapis.dev/python/bigquery/latest/changelog.html#id51 --- packages/pandas-gbq/ci/requirements-3.5.pip | 2 +- packages/pandas-gbq/ci/requirements-3.6-0.20.1.conda | 4 ++-- packages/pandas-gbq/docs/source/changelog.rst | 10 ++++++++-- packages/pandas-gbq/pandas_gbq/gbq.py | 2 +- packages/pandas-gbq/setup.py | 2 +- packages/pandas-gbq/tests/system/test_auth.py | 2 +- packages/pandas-gbq/tests/unit/test_gbq.py | 6 +++--- 7 files changed, 17 insertions(+), 11 deletions(-) diff --git a/packages/pandas-gbq/ci/requirements-3.5.pip b/packages/pandas-gbq/ci/requirements-3.5.pip index 41a4189182c2..7f2e9b251e49 100644 --- a/packages/pandas-gbq/ci/requirements-3.5.pip +++ b/packages/pandas-gbq/ci/requirements-3.5.pip @@ -1,5 +1,5 @@ pandas==0.19.0 google-auth==1.4.1 google-auth-oauthlib==0.0.1 -google-cloud-bigquery==1.9.0 +google-cloud-bigquery==1.11.1 pydata-google-auth==0.1.2 \ No newline at end of file diff --git a/packages/pandas-gbq/ci/requirements-3.6-0.20.1.conda b/packages/pandas-gbq/ci/requirements-3.6-0.20.1.conda index e492c378eb1a..f36e096df9e6 100644 --- a/packages/pandas-gbq/ci/requirements-3.6-0.20.1.conda +++ b/packages/pandas-gbq/ci/requirements-3.6-0.20.1.conda @@ -2,8 +2,8 @@ codecov coverage fastavro flake8 -google-cloud-bigquery==1.9.0 -google-cloud-bigquery-storage==0.5.0 +google-cloud-bigquery==1.11.1 +google-cloud-bigquery-storage pydata-google-auth pytest pytest-cov diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 8695ef11a961..927e281fdcd6 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -10,9 +10,15 @@ Changelog argument to limit the number of rows in the results DataFrame. Set ``max_results`` to 0 to ignore query outputs, such as for DML or DDL queries. (:issue:`102`) +- Add ``progress_bar_type`` argument to :func:`~pandas_gbq.read_gbq()`. Use + this argument to display a progress bar when downloading data. + (:issue:`182`) -- Add ``progress_bar_type`` argument to :func:`~pandas_gbq.read_gbq()`. Use this - argument to display a progress bar when downloading data. (:issue:`182`) +Dependency updates +~~~~~~~~~~~~~~~~~~ + +- Update the minimum version of ``google-cloud-bigquery`` to 1.11.1. + (:issue:`296`) Documentation ~~~~~~~~~~~~~ diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 0da10c4ffd5f..5ee36a270ec6 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -45,7 +45,7 @@ def _check_google_client_version(): raise ImportError("Could not import pkg_resources (setuptools).") # https://github.com/GoogleCloudPlatform/google-cloud-python/blob/master/bigquery/CHANGELOG.md - bigquery_minimum_version = pkg_resources.parse_version("1.9.0") + bigquery_minimum_version = pkg_resources.parse_version("1.11.0") bigquery_client_info_version = pkg_resources.parse_version( BIGQUERY_CLIENT_INFO_VERSION ) diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index f9a0a4319db9..0b97bbfd5315 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -22,7 +22,7 @@ def readme(): "pydata-google-auth", "google-auth", "google-auth-oauthlib", - "google-cloud-bigquery>=1.9.0", + "google-cloud-bigquery>=1.11.1", ] extras = {"tqdm": "tqdm>=4.23.0"} diff --git a/packages/pandas-gbq/tests/system/test_auth.py b/packages/pandas-gbq/tests/system/test_auth.py index 980fbb48ce2e..61dcf96d36a8 100644 --- a/packages/pandas-gbq/tests/system/test_auth.py +++ b/packages/pandas-gbq/tests/system/test_auth.py @@ -74,7 +74,7 @@ def test_get_service_account_credentials_private_key_path(private_key_path): def test_get_service_account_credentials_private_key_contents( - private_key_contents + private_key_contents, ): from google.auth.credentials import Credentials diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index f213f0169dcf..384602a9cf15 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -31,7 +31,7 @@ def _make_connector(project_id="some-project", **kwargs): def min_bq_version(): import pkg_resources - return pkg_resources.parse_version("1.9.0") + return pkg_resources.parse_version("1.11.0") def mock_get_credentials_no_project(*args, **kwargs): @@ -327,7 +327,7 @@ def test_read_gbq_with_inferred_project_id_from_service_account_credentials( def test_read_gbq_without_inferred_project_id_from_compute_engine_credentials( - mock_compute_engine_credentials + mock_compute_engine_credentials, ): with pytest.raises(ValueError, match="Could not determine project ID"): gbq.read_gbq( @@ -406,7 +406,7 @@ def test_read_gbq_with_verbose_new_pandas_warns_deprecation(min_bq_version): def test_read_gbq_with_not_verbose_new_pandas_warns_deprecation( - min_bq_version + min_bq_version, ): import pkg_resources From ad28091864de91ef6fb9a1f47d3031845a031ae5 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 25 Nov 2019 14:10:39 -0800 Subject: [PATCH 199/519] FIX: close BigQuery Storage client transport channel after use (#295) * FIX: close BigQuery Storage client transport channel after use This fixes a file descriptor leak. * blacken --- packages/pandas-gbq/noxfile.py | 7 ++++-- packages/pandas-gbq/pandas_gbq/gbq.py | 36 +++++++++++++++------------ 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index f0eadfd109df..64c0454fcc4f 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -13,10 +13,13 @@ supported_pythons = ["3.5", "3.6", "3.7"] latest_python = "3.7" +# Use a consistent version of black so CI is deterministic. +black_package = "black==19.10b0" + @nox.session def lint(session, python=latest_python): - session.install("black", "flake8") + session.install(black_package, "flake8") session.install("-e", ".") session.run("flake8", "pandas_gbq") session.run("flake8", "tests") @@ -25,7 +28,7 @@ def lint(session, python=latest_python): @nox.session(python=latest_python) def blacken(session): - session.install("black") + session.install(black_package) session.run("black", ".") diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 5ee36a270ec6..ca577de6ab6c 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -370,9 +370,7 @@ def __init__( context.project = self.project_id self.client = self.get_client() - self.bqstorage_client = _make_bqstorage_client( - use_bqstorage_api, self.credentials - ) + self.use_bqstorage_api = use_bqstorage_api # BQ Queries costs $5 per TB. First 1 TB per month is free # see here for more: https://cloud.google.com/bigquery/pricing @@ -541,29 +539,35 @@ def _download_results( if max_results == 0: return None - if max_results is None: - # Only use the BigQuery Storage API if the full result set is requested. - bqstorage_client = self.bqstorage_client - else: + try: bqstorage_client = None + if max_results is None: + # Only use the BigQuery Storage API if the full result set is requested. + bqstorage_client = _make_bqstorage_client( + self.use_bqstorage_api, self.credentials + ) - try: query_job.result() # Get the table schema, so that we can list rows. destination = self.client.get_table(query_job.destination) rows_iter = self.client.list_rows( destination, max_results=max_results ) + + schema_fields = [field.to_api_repr() for field in rows_iter.schema] + nullsafe_dtypes = _bqschema_to_nullsafe_dtypes(schema_fields) + df = rows_iter.to_dataframe( + dtypes=nullsafe_dtypes, + bqstorage_client=bqstorage_client, + progress_bar_type=progress_bar_type, + ) except self.http_error as ex: self.process_http_error(ex) - - schema_fields = [field.to_api_repr() for field in rows_iter.schema] - nullsafe_dtypes = _bqschema_to_nullsafe_dtypes(schema_fields) - df = rows_iter.to_dataframe( - dtypes=nullsafe_dtypes, - bqstorage_client=bqstorage_client, - progress_bar_type=progress_bar_type, - ) + finally: + if bqstorage_client: + # Clean up open socket resources. See: + # https://github.com/pydata/pandas-gbq/issues/294 + bqstorage_client.transport.channel.close() if df.empty: df = _cast_empty_df_dtypes(schema_fields, df) From 215f6a25305b14175be87820e2aaa67f5d8c6d47 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 25 Nov 2019 14:13:28 -0800 Subject: [PATCH 200/519] Release 0.12.0 --- packages/pandas-gbq/docs/source/changelog.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 927e281fdcd6..aee9a7cf0d73 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -3,8 +3,8 @@ Changelog .. _changelog-0.12.0: -0.12.0 / TBD ------------- +0.12.0 / 2019-11-25 +------------------- - Add ``max_results`` argument to :func:`~pandas_gbq.read_gbq()`. Use this argument to limit the number of rows in the results DataFrame. Set From 1d0b5586a8fb61f00764c410d055c8e6a11e086c Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 12 Dec 2019 14:40:58 -0800 Subject: [PATCH 201/519] CLN: remove deprecated `private_key` auth logic (#302) * CLN: remove deprecated private_key auth logic * remove unnecessary deprecation warning for private_key * update docstrings for deprecated private_key arg --- packages/pandas-gbq/docs/source/changelog.rst | 9 + packages/pandas-gbq/noxfile.py | 2 +- packages/pandas-gbq/pandas_gbq/auth.py | 55 +---- packages/pandas-gbq/pandas_gbq/gbq.py | 58 ++--- packages/pandas-gbq/pandas_gbq/load.py | 9 +- packages/pandas-gbq/tests/system/test_auth.py | 28 +-- packages/pandas-gbq/tests/unit/test_auth.py | 59 ++--- packages/pandas-gbq/tests/unit/test_gbq.py | 202 ++++++------------ 8 files changed, 114 insertions(+), 308 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index aee9a7cf0d73..0737247c7e22 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,6 +1,15 @@ Changelog ========= +.. _changelog-0.13.0: + +0.13.0 / 2019-12-12 +------------------- + +- Raise ``NotImplementedError`` when the deprecated ``private_key`` argument + is used. (:issue:`301`) + + .. _changelog-0.12.0: 0.12.0 / 2019-11-25 diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 64c0454fcc4f..1ab4ce898544 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -51,7 +51,7 @@ def unit(session): @nox.session def cover(session, python=latest_python): session.install("coverage", "pytest-cov") - session.run("coverage", "report", "--show-missing", "--fail-under=74") + session.run("coverage", "report", "--show-missing", "--fail-under=73") session.run("coverage", "erase") diff --git a/packages/pandas-gbq/pandas_gbq/auth.py b/packages/pandas-gbq/pandas_gbq/auth.py index bd842d086bdc..448367b42b8c 100644 --- a/packages/pandas-gbq/pandas_gbq/auth.py +++ b/packages/pandas-gbq/pandas_gbq/auth.py @@ -1,11 +1,6 @@ """Private module for fetching Google BigQuery credentials.""" -import json import logging -import os -import os.path - -import pandas_gbq.exceptions logger = logging.getLogger(__name__) @@ -36,7 +31,13 @@ def get_credentials( import pydata_google_auth if private_key: - return get_service_account_credentials(private_key) + raise NotImplementedError( + """The private_key argument is deprecated. Construct a credentials +object, instead, by using the +google.oauth2.service_account.Credentials.from_service_account_file or +google.oauth2.service_account.Credentials.from_service_account_info class +method from the google-auth package.""" + ) credentials, default_project_id = pydata_google_auth.default( SCOPES, @@ -50,48 +51,6 @@ def get_credentials( return credentials, project_id -def get_service_account_credentials(private_key): - """DEPRECATED: Load service account credentials from key data or key path.""" - - import google.auth.transport.requests - from google.oauth2.service_account import Credentials - - is_path = os.path.isfile(private_key) - - try: - if is_path: - with open(private_key) as f: - json_key = json.loads(f.read()) - else: - # ugly hack: 'private_key' field has new lines inside, - # they break json parser, but we need to preserve them - json_key = json.loads(private_key.replace("\n", " ")) - json_key["private_key"] = json_key["private_key"].replace( - " ", "\n" - ) - - json_key["private_key"] = bytes(json_key["private_key"], "UTF-8") - credentials = Credentials.from_service_account_info(json_key) - credentials = credentials.with_scopes(SCOPES) - - # Refresh the token before trying to use it. - request = google.auth.transport.requests.Request() - credentials.refresh(request) - - return credentials, json_key.get("project_id") - except (KeyError, ValueError, TypeError, AttributeError): - raise pandas_gbq.exceptions.InvalidPrivateKeyFormat( - "Detected private_key as {}. ".format( - "path" if is_path else "contents" - ) - + "Private key is missing or invalid. It should be service " - "account private key JSON (file path or string contents) " - 'with at least two keys: "client_email" and "private_key". ' - "Can be obtained from: https://console.developers.google." - "com/permissions/serviceaccounts" - ) - - def get_credentials_cache(reauth,): import pydata_google_auth.cache diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index ca577de6ab6c..f254d528b254 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -20,14 +20,6 @@ BIGQUERY_INSTALLED_VERSION = None BIGQUERY_CLIENT_INFO_VERSION = "1.12.0" HAS_CLIENT_INFO = False -SHOW_VERBOSE_DEPRECATION = False -SHOW_PRIVATE_KEY_DEPRECATION = False -PRIVATE_KEY_DEPRECATION_MESSAGE = ( - "private_key is deprecated and will be removed in a future version." - "Use the credentials argument instead. See " - "https://pandas-gbq.readthedocs.io/en/latest/howto/authentication.html " - "for examples on using the credentials argument with service account keys." -) try: import tqdm # noqa @@ -36,7 +28,7 @@ def _check_google_client_version(): - global BIGQUERY_INSTALLED_VERSION, HAS_CLIENT_INFO, SHOW_VERBOSE_DEPRECATION, SHOW_PRIVATE_KEY_DEPRECATION + global BIGQUERY_INSTALLED_VERSION, HAS_CLIENT_INFO, SHOW_VERBOSE_DEPRECATION try: import pkg_resources @@ -74,10 +66,6 @@ def _check_google_client_version(): SHOW_VERBOSE_DEPRECATION = ( pandas_installed_version >= pandas_version_wo_verbosity ) - pandas_version_with_credentials_arg = pkg_resources.parse_version("0.24.0") - SHOW_PRIVATE_KEY_DEPRECATION = ( - pandas_installed_version >= pandas_version_with_credentials_arg - ) def _test_google_api_imports(): @@ -951,27 +939,12 @@ def read_gbq( results. .. versionadded:: 0.12.0 - verbose : None, deprecated - Deprecated in Pandas-GBQ 0.4.0. Use the `logging module - to adjust verbosity instead - `__. - private_key : str, deprecated - Deprecated in pandas-gbq version 0.8.0. Use the ``credentials`` - parameter and - :func:`google.oauth2.service_account.Credentials.from_service_account_info` - or - :func:`google.oauth2.service_account.Credentials.from_service_account_file` - instead. - - Service account private key in JSON format. Can be file path - or string contents. This is useful for remote server - authentication (eg. Jupyter/IPython notebook on remote host). - progress_bar_type (Optional[str]): - If set, use the `tqdm `_ library to + If set, use the `tqdm `__ library to display a progress bar while the data downloads. Install the ``tqdm`` package to use this feature. Possible values of ``progress_bar_type`` include: + ``None`` No progress bar. ``'tqdm'`` @@ -983,6 +956,17 @@ def read_gbq( ``'tqdm_gui'`` Use the :func:`tqdm.tqdm_gui` function to display a progress bar as a graphical dialog box. + verbose : None, deprecated + Deprecated in Pandas-GBQ 0.4.0. Use the `logging module + to adjust verbosity instead + `__. + private_key : str, deprecated + Deprecated in pandas-gbq version 0.8.0. Use the ``credentials`` + parameter and + :func:`google.oauth2.service_account.Credentials.from_service_account_info` + or + :func:`google.oauth2.service_account.Credentials.from_service_account_file` + instead. Returns ------- @@ -1008,11 +992,6 @@ def read_gbq( stacklevel=2, ) - if private_key is not None and SHOW_PRIVATE_KEY_DEPRECATION: - warnings.warn( - PRIVATE_KEY_DEPRECATION_MESSAGE, FutureWarning, stacklevel=2 - ) - if dialect not in ("legacy", "standard"): raise ValueError("'{0}' is not valid for dialect".format(dialect)) @@ -1172,10 +1151,6 @@ def to_gbq( or :func:`google.oauth2.service_account.Credentials.from_service_account_file` instead. - - Service account private key in JSON format. Can be file path - or string contents. This is useful for remote server - authentication (eg. Jupyter/IPython notebook on remote host). """ _test_google_api_imports() @@ -1190,11 +1165,6 @@ def to_gbq( stacklevel=1, ) - if private_key is not None and SHOW_PRIVATE_KEY_DEPRECATION: - warnings.warn( - PRIVATE_KEY_DEPRECATION_MESSAGE, FutureWarning, stacklevel=2 - ) - if if_exists not in ("fail", "replace", "append"): raise ValueError("'{0}' is not valid for if_exists".format(if_exists)) diff --git a/packages/pandas-gbq/pandas_gbq/load.py b/packages/pandas-gbq/pandas_gbq/load.py index 3e9d570e6e85..04b32efaf983 100644 --- a/packages/pandas-gbq/pandas_gbq/load.py +++ b/packages/pandas-gbq/pandas_gbq/load.py @@ -1,6 +1,7 @@ """Helper methods for loading data into BigQuery""" -import six +import io + from google.cloud import bigquery import pandas_gbq.schema @@ -12,7 +13,7 @@ def encode_chunk(dataframe): Args: dataframe (pandas.DataFrame): A chunk of a dataframe to encode """ - csv_buffer = six.StringIO() + csv_buffer = io.StringIO() dataframe.to_csv( csv_buffer, index=False, @@ -25,10 +26,8 @@ def encode_chunk(dataframe): # Convert to a BytesIO buffer so that unicode text is properly handled. # See: https://github.com/pydata/pandas-gbq/issues/106 body = csv_buffer.getvalue() - if isinstance(body, bytes): - body = body.decode("utf-8") body = body.encode("utf-8") - return six.BytesIO(body) + return io.BytesIO(body) def encode_chunks(dataframe, chunksize=None): diff --git a/packages/pandas-gbq/tests/system/test_auth.py b/packages/pandas-gbq/tests/system/test_auth.py index 61dcf96d36a8..eef18f96065a 100644 --- a/packages/pandas-gbq/tests/system/test_auth.py +++ b/packages/pandas-gbq/tests/system/test_auth.py @@ -1,5 +1,6 @@ """System tests for fetching Google BigQuery credentials.""" +import os from unittest import mock import pytest @@ -57,34 +58,11 @@ def _check_if_can_get_correct_default_credentials(): def test_should_be_able_to_get_valid_credentials(project_id, private_key_path): - credentials, _ = auth.get_credentials( - project_id=project_id, private_key=private_key_path - ) + os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = private_key_path + credentials, _ = auth.get_credentials(project_id=project_id) assert credentials.valid -def test_get_service_account_credentials_private_key_path(private_key_path): - from google.auth.credentials import Credentials - - credentials, project_id = auth.get_service_account_credentials( - private_key_path - ) - assert isinstance(credentials, Credentials) - assert _try_credentials(project_id, credentials) is not None - - -def test_get_service_account_credentials_private_key_contents( - private_key_contents, -): - from google.auth.credentials import Credentials - - credentials, project_id = auth.get_service_account_credentials( - private_key_contents - ) - assert isinstance(credentials, Credentials) - assert _try_credentials(project_id, credentials) is not None - - @pytest.mark.local_auth def test_get_credentials_bad_file_returns_user_credentials( project_id, monkeypatch diff --git a/packages/pandas-gbq/tests/unit/test_auth.py b/packages/pandas-gbq/tests/unit/test_auth.py index d0a7cbf6cddd..4f1e76d90c97 100644 --- a/packages/pandas-gbq/tests/unit/test_auth.py +++ b/packages/pandas-gbq/tests/unit/test_auth.py @@ -1,28 +1,14 @@ # -*- coding: utf-8 -*- import json -import os.path - -from pandas_gbq import auth - from unittest import mock +import pytest -def test_get_credentials_private_key_contents(monkeypatch): - from google.oauth2 import service_account +from pandas_gbq import auth - @classmethod - def from_service_account_info(cls, key_info): - mock_credentials = mock.create_autospec(cls) - mock_credentials.with_scopes.return_value = mock_credentials - mock_credentials.refresh.return_value = mock_credentials - return mock_credentials - monkeypatch.setattr( - service_account.Credentials, - "from_service_account_info", - from_service_account_info, - ) +def test_get_credentials_private_key_raises_notimplementederror(monkeypatch): private_key = json.dumps( { "private_key": "some_key", @@ -30,34 +16,8 @@ def from_service_account_info(cls, key_info): "project_id": "private-key-project", } ) - credentials, project = auth.get_credentials(private_key=private_key) - - assert credentials is not None - assert project == "private-key-project" - - -def test_get_credentials_private_key_path(monkeypatch): - from google.oauth2 import service_account - - @classmethod - def from_service_account_info(cls, key_info): - mock_credentials = mock.create_autospec(cls) - mock_credentials.with_scopes.return_value = mock_credentials - mock_credentials.refresh.return_value = mock_credentials - return mock_credentials - - monkeypatch.setattr( - service_account.Credentials, - "from_service_account_info", - from_service_account_info, - ) - private_key = os.path.join( - os.path.dirname(__file__), "..", "data", "dummy_key.json" - ) - credentials, project = auth.get_credentials(private_key=private_key) - - assert credentials is not None - assert project is None + with pytest.raises(NotImplementedError, match="private_key"): + auth.get_credentials(private_key=private_key) def test_get_credentials_default_credentials(monkeypatch): @@ -101,3 +61,12 @@ def mock_default_credentials(scopes=None, request=None): credentials, project = auth.get_credentials() assert project is None assert credentials is mock_user_credentials + + +def test_get_credentials_cache_w_reauth(): + import pydata_google_auth.cache + + cache = auth.get_credentials_cache(True) + assert isinstance( + cache, pydata_google_auth.cache.WriteOnlyCredentialsCache + ) diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index 384602a9cf15..e0d0c8a407d2 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -7,11 +7,9 @@ import numpy import pandas from pandas import DataFrame -import pandas.util.testing as tm import pkg_resources import pytest -import pandas_gbq.exceptions from pandas_gbq import gbq @@ -224,59 +222,14 @@ def test_to_gbq_with_verbose_old_pandas_no_warnings(recwarn, min_bq_version): assert len(recwarn) == 0 -@pytest.mark.skipif( - pandas_installed_version < pkg_resources.parse_version("0.24.0"), - reason="Requires pandas 0.24+", -) -def test_to_gbq_with_private_key_new_pandas_warns_deprecation( - min_bq_version, monkeypatch -): - import pkg_resources - from pandas_gbq import auth - - monkeypatch.setattr(auth, "get_credentials", mock_get_credentials) - - pandas_version = pkg_resources.parse_version("0.24.0") - with pytest.warns(FutureWarning), mock.patch( - "pkg_resources.Distribution.parsed_version", - new_callable=mock.PropertyMock, - ) as mock_version: - mock_version.side_effect = [min_bq_version, pandas_version] - try: - gbq.to_gbq( - DataFrame([[1]]), - "dataset.tablename", - project_id="my-project", - private_key="path/to/key.json", - ) - except gbq.TableCreationError: - pass - - -def test_to_gbq_with_private_key_old_pandas_no_warnings( - recwarn, min_bq_version, monkeypatch -): - import pkg_resources - from pandas_gbq import auth - - monkeypatch.setattr(auth, "get_credentials", mock_get_credentials) - - pandas_version = pkg_resources.parse_version("0.23.4") - with mock.patch( - "pkg_resources.Distribution.parsed_version", - new_callable=mock.PropertyMock, - ) as mock_version: - mock_version.side_effect = [min_bq_version, pandas_version] - try: - gbq.to_gbq( - DataFrame([[1]]), - "dataset.tablename", - project_id="my-project", - private_key="path/to/key.json", - ) - except gbq.TableCreationError: - pass - assert len(recwarn) == 0 +def test_to_gbq_with_private_key_raises_notimplementederror(): + with pytest.raises(NotImplementedError, match="private_key"): + gbq.to_gbq( + DataFrame([[1]]), + "dataset.tablename", + project_id="my-project", + private_key="path/to/key.json", + ) def test_to_gbq_doesnt_run_query( @@ -292,6 +245,31 @@ def test_to_gbq_doesnt_run_query( mock_bigquery_client.query.assert_not_called() +def test_to_gbq_w_empty_df(mock_bigquery_client): + import google.api_core.exceptions + + mock_bigquery_client.get_table.side_effect = google.api_core.exceptions.NotFound( + "my_table" + ) + gbq.to_gbq(DataFrame(), "my_dataset.my_table", project_id="1234") + mock_bigquery_client.create_table.assert_called_with(mock.ANY) + mock_bigquery_client.load_table_from_dataframe.assert_not_called() + mock_bigquery_client.load_table_from_file.assert_not_called() + + +def test_to_gbq_creates_dataset(mock_bigquery_client): + import google.api_core.exceptions + + mock_bigquery_client.get_table.side_effect = google.api_core.exceptions.NotFound( + "my_table" + ) + mock_bigquery_client.get_dataset.side_effect = google.api_core.exceptions.NotFound( + "my_dataset" + ) + gbq.to_gbq(DataFrame([[1]]), "my_dataset.my_table", project_id="1234") + mock_bigquery_client.create_dataset.assert_called_with(mock.ANY) + + def test_read_gbq_with_no_project_id_given_should_fail(monkeypatch): import pydata_google_auth @@ -348,51 +326,6 @@ def test_read_gbq_with_max_results_ten(monkeypatch, mock_bigquery_client): mock_bigquery_client.list_rows.assert_called_with(mock.ANY, max_results=10) -def test_read_gbq_with_invalid_private_key_json_should_fail(): - with pytest.raises(pandas_gbq.exceptions.InvalidPrivateKeyFormat): - gbq.read_gbq( - "SELECT 1", dialect="standard", project_id="x", private_key="y" - ) - - -def test_read_gbq_with_empty_private_key_json_should_fail(): - with pytest.raises(pandas_gbq.exceptions.InvalidPrivateKeyFormat): - gbq.read_gbq( - "SELECT 1", dialect="standard", project_id="x", private_key="{}" - ) - - -def test_read_gbq_with_private_key_json_wrong_types_should_fail(): - with pytest.raises(pandas_gbq.exceptions.InvalidPrivateKeyFormat): - gbq.read_gbq( - "SELECT 1", - dialect="standard", - project_id="x", - private_key='{ "client_email" : 1, "private_key" : True }', - ) - - -def test_read_gbq_with_empty_private_key_file_should_fail(): - with tm.ensure_clean() as empty_file_path: - with pytest.raises(pandas_gbq.exceptions.InvalidPrivateKeyFormat): - gbq.read_gbq( - "SELECT 1", - dialect="standard", - project_id="x", - private_key=empty_file_path, - ) - - -def test_read_gbq_with_corrupted_private_key_json_should_fail(): - with pytest.raises(pandas_gbq.exceptions.InvalidPrivateKeyFormat): - gbq.read_gbq( - "SELECT 1", - dialect="standard", - project_id="x", - private_key="99999999999999999", - ) - - def test_read_gbq_with_verbose_new_pandas_warns_deprecation(min_bq_version): import pkg_resources @@ -432,56 +365,24 @@ def test_read_gbq_wo_verbose_w_new_pandas_no_warnings(recwarn, min_bq_version): assert len(recwarn) == 0 -def test_read_gbq_with_verbose_old_pandas_no_warnings(recwarn, min_bq_version): - import pkg_resources - - pandas_version = pkg_resources.parse_version("0.22.0") - with mock.patch( - "pkg_resources.Distribution.parsed_version", - new_callable=mock.PropertyMock, - ) as mock_version: - mock_version.side_effect = [min_bq_version, pandas_version] - gbq.read_gbq( - "SELECT 1", - project_id="my-project", - dialect="standard", - verbose=True, - ) - assert len(recwarn) == 0 - - -@pytest.mark.skipif( - pandas_installed_version < pkg_resources.parse_version("0.24.0"), - reason="Requires pandas 0.24+", -) -def test_read_gbq_with_private_key_new_pandas_warns_deprecation( - min_bq_version, monkeypatch -): +def test_read_gbq_with_old_bq_raises_importerror(): import pkg_resources - from pandas_gbq import auth - - monkeypatch.setattr(auth, "get_credentials", mock_get_credentials) - pandas_version = pkg_resources.parse_version("0.24.0") - with pytest.warns(FutureWarning), mock.patch( + bigquery_version = pkg_resources.parse_version("0.27.0") + with pytest.raises(ImportError, match="google-cloud-bigquery"), mock.patch( "pkg_resources.Distribution.parsed_version", new_callable=mock.PropertyMock, ) as mock_version: - mock_version.side_effect = [min_bq_version, pandas_version] + mock_version.side_effect = [bigquery_version] gbq.read_gbq( - "SELECT 1", project_id="my-project", private_key="path/to/key.json" + "SELECT 1", project_id="my-project", ) -def test_read_gbq_with_private_key_old_pandas_no_warnings( - recwarn, min_bq_version, monkeypatch -): +def test_read_gbq_with_verbose_old_pandas_no_warnings(recwarn, min_bq_version): import pkg_resources - from pandas_gbq import auth - - monkeypatch.setattr(auth, "get_credentials", mock_get_credentials) - pandas_version = pkg_resources.parse_version("0.23.4") + pandas_version = pkg_resources.parse_version("0.22.0") with mock.patch( "pkg_resources.Distribution.parsed_version", new_callable=mock.PropertyMock, @@ -491,16 +392,37 @@ def test_read_gbq_with_private_key_old_pandas_no_warnings( "SELECT 1", project_id="my-project", dialect="standard", - private_key="path/to/key.json", + verbose=True, ) assert len(recwarn) == 0 +def test_read_gbq_with_private_raises_notimplmentederror(): + with pytest.raises(NotImplementedError, match="private_key"): + gbq.read_gbq( + "SELECT 1", project_id="my-project", private_key="path/to/key.json" + ) + + def test_read_gbq_with_invalid_dialect(): with pytest.raises(ValueError, match="is not valid for dialect"): gbq.read_gbq("SELECT 1", dialect="invalid") +def test_read_gbq_with_configuration_query(): + df = gbq.read_gbq(None, configuration={"query": {"query": "SELECT 2"}}) + assert df is not None + + +def test_read_gbq_with_configuration_duplicate_query_raises_error(): + with pytest.raises( + ValueError, match="Query statement can't be specified inside config" + ): + gbq.read_gbq( + "SELECT 1", configuration={"query": {"query": "SELECT 2"}} + ) + + def test_generate_bq_schema_deprecated(): # 11121 Deprecation of generate_bq_schema with pytest.warns(FutureWarning): From ff24fff8fdfd55514b80502af4e4259003ecc5a8 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 12 Feb 2020 16:32:14 -0600 Subject: [PATCH 202/519] TST: use larger query for timeout test (#311) * TST: use larger query for timeout test Also, allow jobTimeoutMs, as that is the name for the backend parameter. * blacken * FIX: No module named 'numpy.testing.decorators' * install pre-release pandas _after_ install dependencies --- packages/pandas-gbq/ci/requirements-3.5.pip | 1 + packages/pandas-gbq/ci/run_conda.sh | 5 ++- packages/pandas-gbq/pandas_gbq/gbq.py | 5 ++- packages/pandas-gbq/tests/system/test_gbq.py | 43 ++++++++++++++------ 4 files changed, 39 insertions(+), 15 deletions(-) diff --git a/packages/pandas-gbq/ci/requirements-3.5.pip b/packages/pandas-gbq/ci/requirements-3.5.pip index 7f2e9b251e49..2f9911de8262 100644 --- a/packages/pandas-gbq/ci/requirements-3.5.pip +++ b/packages/pandas-gbq/ci/requirements-3.5.pip @@ -1,3 +1,4 @@ +numpy==1.13.3 pandas==0.19.0 google-auth==1.4.1 google-auth-oauthlib==0.0.1 diff --git a/packages/pandas-gbq/ci/run_conda.sh b/packages/pandas-gbq/ci/run_conda.sh index d0b892aac490..526e6de630eb 100755 --- a/packages/pandas-gbq/ci/run_conda.sh +++ b/packages/pandas-gbq/ci/run_conda.sh @@ -11,6 +11,9 @@ conda update -q conda conda info -a conda create -q -n test-environment python=$PYTHON source activate test-environment +REQ="ci/requirements-${PYTHON}-${PANDAS}" +conda install -q --file "$REQ.conda"; + if [[ "$PANDAS" == "NIGHTLY" ]]; then conda install -q numpy pytz python-dateutil; PRE_WHEELS="https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com"; @@ -19,8 +22,6 @@ else conda install -q pandas=$PANDAS; fi -REQ="ci/requirements-${PYTHON}-${PANDAS}" -conda install -q --file "$REQ.conda"; python setup.py develop --no-deps # Run the tests diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index f254d528b254..d5b26a1aaba7 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -480,7 +480,10 @@ def run_query( while query_reply.state != "DONE": self.log_elapsed_seconds(" Elapsed", "s. Waiting...") - timeout_ms = job_config["query"].get("timeoutMs") + timeout_ms = job_config.get("jobTimeoutMs") or job_config[ + "query" + ].get("timeoutMs") + timeout_ms = int(timeout_ms) if timeout_ms else None if timeout_ms and timeout_ms < self.get_elapsed_seconds() * 1000: raise QueryTimeout("Query timeout: {} ms".format(timeout_ms)) diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index 729cc7ce1f86..034e7c8d99ec 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -211,7 +211,10 @@ def test_should_properly_handle_null_integers(self, project_id): credentials=self.credentials, dialect="legacy", ) - tm.assert_frame_equal(df, DataFrame({"null_integer": [None]})) + tm.assert_frame_equal( + df, + DataFrame({"null_integer": pandas.Series([None], dtype="object")}), + ) def test_should_properly_handle_valid_floats(self, project_id): from math import pi @@ -772,17 +775,33 @@ def test_configuration_raises_value_error_with_multiple_config( ) def test_timeout_configuration(self, project_id): - sql_statement = "SELECT 1" - config = {"query": {"timeoutMs": 1}} - # Test that QueryTimeout error raises - with pytest.raises(gbq.QueryTimeout): - gbq.read_gbq( - sql_statement, - project_id=project_id, - credentials=self.credentials, - configuration=config, - dialect="legacy", - ) + sql_statement = """ + SELECT + SUM(bottles_sold) total_bottles, + UPPER(category_name) category_name, + magnitude, + liquor.zip_code zip_code + FROM `bigquery-public-data.iowa_liquor_sales.sales` liquor + JOIN `bigquery-public-data.geo_us_boundaries.zip_codes` zip_codes + ON liquor.zip_code = zip_codes.zip_code + JOIN `bigquery-public-data.noaa_historic_severe_storms.tornado_paths` tornados + ON liquor.date = tornados.storm_date + WHERE ST_INTERSECTS(tornado_path_geom, zip_code_geom) + GROUP BY category_name, magnitude, zip_code + ORDER BY magnitude ASC, total_bottles DESC + """ + configs = [ + {"query": {"useQueryCache": False, "timeoutMs": 1}}, + {"query": {"useQueryCache": False}, "jobTimeoutMs": 1}, + ] + for config in configs: + with pytest.raises(gbq.QueryTimeout): + gbq.read_gbq( + sql_statement, + project_id=project_id, + credentials=self.credentials, + configuration=config, + ) def test_query_response_bytes(self): assert self.gbq_connector.sizeof_fmt(999) == "999.0 B" From 0dae1df81de19d793d15e23db29e211eaac34a7b Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 13 Feb 2020 10:57:36 -0600 Subject: [PATCH 203/519] BUG: fix AttributeError with BQ Storage API to download empty results (#310) * BUG: fix AttributeError with BQ Storage API to download empty results Refactors timestamp helpers to their own file to help reduce the size of the gbq module. * blacken * fix lint * fix test_zero_rows * update release date --- packages/pandas-gbq/docs/source/changelog.rst | 9 ++ packages/pandas-gbq/pandas_gbq/gbq.py | 30 ++--- packages/pandas-gbq/pandas_gbq/timestamp.py | 40 +++++++ packages/pandas-gbq/tests/system/conftest.py | 78 ++++++++++++ packages/pandas-gbq/tests/system/test_gbq.py | 112 ------------------ .../system/test_read_gbq_with_bqstorage.py | 69 +++++++++++ .../pandas-gbq/tests/unit/test_timestamp.py | 72 +++++++++++ 7 files changed, 275 insertions(+), 135 deletions(-) create mode 100644 packages/pandas-gbq/pandas_gbq/timestamp.py create mode 100644 packages/pandas-gbq/tests/system/conftest.py create mode 100644 packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py create mode 100644 packages/pandas-gbq/tests/unit/test_timestamp.py diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 0737247c7e22..56586562029d 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,6 +1,15 @@ Changelog ========= +.. _changelog-0.13.1: + +0.13.1 / 2020-02-13 +------------------- + +- Fix ``AttributeError`` with BQ Storage API to download empty results. + (:issue:`299`) + + .. _changelog-0.13.0: 0.13.0 / 2019-12-12 diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index d5b26a1aaba7..db699096c058 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -8,12 +8,14 @@ try: # The BigQuery Storage API client is an optional dependency. It is only # required when use_bqstorage_api=True. - from google.cloud import bigquery_storage + from google.cloud import bigquery_storage_v1beta1 except ImportError: # pragma: NO COVER - bigquery_storage = None + bigquery_storage_v1beta1 = None from pandas_gbq.exceptions import AccessDenied import pandas_gbq.schema +import pandas_gbq.timestamp + logger = logging.getLogger(__name__) @@ -564,7 +566,7 @@ def _download_results( df = _cast_empty_df_dtypes(schema_fields, df) # Ensure any TIMESTAMP columns are tz-aware. - df = _localize_df(schema_fields, df) + df = pandas_gbq.timestamp.localize_df(df, schema_fields) logger.debug("Got {} rows.\n".format(rows_iter.total_rows)) return df @@ -784,29 +786,11 @@ def _cast_empty_df_dtypes(schema_fields, df): return df -def _localize_df(schema_fields, df): - """Localize any TIMESTAMP columns to tz-aware type. - - In pandas versions before 0.24.0, DatetimeTZDtype cannot be used as the - dtype in Series/DataFrame construction, so localize those columns after - the DataFrame is constructed. - """ - for field in schema_fields: - column = str(field["name"]) - if field["mode"].upper() == "REPEATED": - continue - - if field["type"].upper() == "TIMESTAMP" and df[column].dt.tz is None: - df[column] = df[column].dt.tz_localize("UTC") - - return df - - def _make_bqstorage_client(use_bqstorage_api, credentials): if not use_bqstorage_api: return None - if bigquery_storage is None: + if bigquery_storage_v1beta1 is None: raise ImportError( "Install the google-cloud-bigquery-storage and fastavro/pyarrow " "packages to use the BigQuery Storage API." @@ -818,7 +802,7 @@ def _make_bqstorage_client(use_bqstorage_api, credentials): client_info = google.api_core.gapic_v1.client_info.ClientInfo( user_agent="pandas-{}".format(pandas.__version__) ) - return bigquery_storage.BigQueryStorageClient( + return bigquery_storage_v1beta1.BigQueryStorageClient( credentials=credentials, client_info=client_info ) diff --git a/packages/pandas-gbq/pandas_gbq/timestamp.py b/packages/pandas-gbq/pandas_gbq/timestamp.py new file mode 100644 index 000000000000..8bdcfa427205 --- /dev/null +++ b/packages/pandas-gbq/pandas_gbq/timestamp.py @@ -0,0 +1,40 @@ +"""Helpers for working with TIMESTAMP data type. + +Private module. +""" + + +def localize_df(df, schema_fields): + """Localize any TIMESTAMP columns to tz-aware type. + + In pandas versions before 0.24.0, DatetimeTZDtype cannot be used as the + dtype in Series/DataFrame construction, so localize those columns after + the DataFrame is constructed. + + Parameters + ---------- + schema_fields: sequence of dict + BigQuery schema in parsed JSON data format. + df: pandaas.DataFrame + DataFrame in which to localize TIMESTAMP columns. + + + Returns + ------- + pandas.DataFrame + DataFrame with localized TIMESTAMP columns. + """ + if len(df.index) == 0: + # If there are no rows, there is nothing to do. + # Fix for https://github.com/pydata/pandas-gbq/issues/299 + return df + + for field in schema_fields: + column = str(field["name"]) + if "mode" in field and field["mode"].upper() == "REPEATED": + continue + + if field["type"].upper() == "TIMESTAMP" and df[column].dt.tz is None: + df[column] = df[column].dt.tz_localize("UTC") + + return df diff --git a/packages/pandas-gbq/tests/system/conftest.py b/packages/pandas-gbq/tests/system/conftest.py new file mode 100644 index 000000000000..2004519ac97d --- /dev/null +++ b/packages/pandas-gbq/tests/system/conftest.py @@ -0,0 +1,78 @@ +import google.oauth2.service_account +import pytest + + +@pytest.fixture(params=["env"]) +def project(request, project_id): + if request.param == "env": + return project_id + elif request.param == "none": + return None + + +@pytest.fixture() +def credentials(private_key_path): + return google.oauth2.service_account.Credentials.from_service_account_file( + private_key_path + ) + + +@pytest.fixture() +def gbq_connector(project, credentials): + from pandas_gbq import gbq + + return gbq.GbqConnector(project, credentials=credentials) + + +@pytest.fixture() +def random_dataset(bigquery_client, random_dataset_id): + from google.cloud import bigquery + + dataset_ref = bigquery_client.dataset(random_dataset_id) + dataset = bigquery.Dataset(dataset_ref) + bigquery_client.create_dataset(dataset) + return dataset + + +@pytest.fixture() +def tokyo_dataset(bigquery_client, random_dataset_id): + from google.cloud import bigquery + + dataset_ref = bigquery_client.dataset(random_dataset_id) + dataset = bigquery.Dataset(dataset_ref) + dataset.location = "asia-northeast1" + bigquery_client.create_dataset(dataset) + return random_dataset_id + + +@pytest.fixture() +def tokyo_table(bigquery_client, tokyo_dataset): + table_id = "tokyo_table" + # Create a random table using DDL. + # https://github.com/GoogleCloudPlatform/golang-samples/blob/2ab2c6b79a1ea3d71d8f91609b57a8fbde07ae5d/bigquery/snippets/snippet.go#L739 + bigquery_client.query( + """CREATE TABLE {}.{} + AS SELECT + 2000 + CAST(18 * RAND() as INT64) as year, + IF(RAND() > 0.5,"foo","bar") as token + FROM UNNEST(GENERATE_ARRAY(0,5,1)) as r + """.format( + tokyo_dataset, table_id + ), + location="asia-northeast1", + ).result() + return table_id + + +@pytest.fixture() +def gbq_dataset(project, credentials): + from pandas_gbq import gbq + + return gbq._Dataset(project, credentials=credentials) + + +@pytest.fixture() +def gbq_table(project, credentials, random_dataset_id): + from pandas_gbq import gbq + + return gbq._Table(project, random_dataset_id, credentials=credentials) diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index 034e7c8d99ec..785e47101f13 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -1,10 +1,8 @@ # -*- coding: utf-8 -*- import sys -import uuid from datetime import datetime -import google.oauth2.service_account import numpy as np import pandas import pandas.api.types @@ -28,76 +26,6 @@ def test_imports(): gbq._test_google_api_imports() -@pytest.fixture(params=["env"]) -def project(request, project_id): - if request.param == "env": - return project_id - elif request.param == "none": - return None - - -@pytest.fixture() -def credentials(private_key_path): - return google.oauth2.service_account.Credentials.from_service_account_file( - private_key_path - ) - - -@pytest.fixture() -def gbq_connector(project, credentials): - return gbq.GbqConnector(project, credentials=credentials) - - -@pytest.fixture() -def random_dataset(bigquery_client, random_dataset_id): - from google.cloud import bigquery - - dataset_ref = bigquery_client.dataset(random_dataset_id) - dataset = bigquery.Dataset(dataset_ref) - bigquery_client.create_dataset(dataset) - return dataset - - -@pytest.fixture() -def tokyo_dataset(bigquery_client, random_dataset_id): - from google.cloud import bigquery - - dataset_ref = bigquery_client.dataset(random_dataset_id) - dataset = bigquery.Dataset(dataset_ref) - dataset.location = "asia-northeast1" - bigquery_client.create_dataset(dataset) - return random_dataset_id - - -@pytest.fixture() -def tokyo_table(bigquery_client, tokyo_dataset): - table_id = "tokyo_table" - # Create a random table using DDL. - # https://github.com/GoogleCloudPlatform/golang-samples/blob/2ab2c6b79a1ea3d71d8f91609b57a8fbde07ae5d/bigquery/snippets/snippet.go#L739 - bigquery_client.query( - """CREATE TABLE {}.{} - AS SELECT - 2000 + CAST(18 * RAND() as INT64) as year, - IF(RAND() > 0.5,"foo","bar") as token - FROM UNNEST(GENERATE_ARRAY(0,5,1)) as r - """.format( - tokyo_dataset, table_id - ), - location="asia-northeast1", - ).result() - return table_id - - -@pytest.fixture() -def gbq_dataset(project, credentials): - return gbq._Dataset(project, credentials=credentials) - - -@pytest.fixture() -def gbq_table(project, credentials, random_dataset_id): - return gbq._Table(project, random_dataset_id, credentials=credentials) - - def make_mixed_dataframe_v2(test_size): # create df to test for all BQ datatypes except RECORD bools = np.random.randint(2, size=(1, test_size)).astype(bool) @@ -600,9 +528,6 @@ def test_zero_rows(self, project_id): empty_columns, columns=["name", "number", "is_hurricane", "iso_time"], ) - expected_result["iso_time"] = expected_result[ - "iso_time" - ].dt.tz_localize("UTC") tm.assert_frame_equal(df, expected_result, check_index_type=False) def test_one_row_one_column(self, project_id): @@ -917,43 +842,6 @@ def test_tokyo(self, tokyo_dataset, tokyo_table, project_id): assert df["max_year"][0] >= 2000 -@pytest.mark.slow(reason="Large query for BQ Storage API tests.") -def test_read_gbq_w_bqstorage_api(credentials, random_dataset): - pytest.importorskip("google.cloud.bigquery_storage") - df = gbq.read_gbq( - """ - SELECT - total_amount, - passenger_count, - trip_distance - FROM `bigquery-public-data.new_york_taxi_trips.tlc_green_trips_2014` - -- Select non-null rows for no-copy conversion from Arrow to pandas. - WHERE total_amount IS NOT NULL - AND passenger_count IS NOT NULL - AND trip_distance IS NOT NULL - LIMIT 10000000 - """, - use_bqstorage_api=True, - credentials=credentials, - configuration={ - "query": { - "destinationTable": { - "projectId": random_dataset.project, - "datasetId": random_dataset.dataset_id, - "tableId": "".join( - [ - "test_read_gbq_w_bqstorage_api_", - str(uuid.uuid4()).replace("-", "_"), - ] - ), - }, - "writeDisposition": "WRITE_TRUNCATE", - } - }, - ) - assert len(df) == 10000000 - - class TestToGBQIntegration(object): @pytest.fixture(autouse=True, scope="function") def setup(self, project, credentials, random_dataset_id): diff --git a/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py b/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py new file mode 100644 index 000000000000..31f2e48412d1 --- /dev/null +++ b/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py @@ -0,0 +1,69 @@ +"""System tests for read_gbq using the BigQuery Storage API.""" + +import functools +import uuid + +import pytest + + +pytest.importorskip("google.cloud.bigquery_storage_v1beta1") + + +@pytest.fixture +def method_under_test(credentials): + import pandas_gbq + + return functools.partial(pandas_gbq.read_gbq, credentials=credentials) + + +@pytest.mark.parametrize( + "query_string", + ( + ("SELECT * FROM (SELECT 1) WHERE TRUE = FALSE;"), + ( + "SELECT * FROM (SELECT TIMESTAMP('2020-02-11 16:33:32-06:00')) WHERE TRUE = FALSE;" + ), + ), +) +def test_empty_results(method_under_test, query_string): + """Test with an empty dataframe. + + See: https://github.com/pydata/pandas-gbq/issues/299 + """ + df = method_under_test(query_string, use_bqstorage_api=True,) + assert len(df.index) == 0 + + +@pytest.mark.slow(reason="Large query for BQ Storage API tests.") +def test_large_results(random_dataset, method_under_test): + df = method_under_test( + """ + SELECT + total_amount, + passenger_count, + trip_distance + FROM `bigquery-public-data.new_york_taxi_trips.tlc_green_trips_2014` + -- Select non-null rows for no-copy conversion from Arrow to pandas. + WHERE total_amount IS NOT NULL + AND passenger_count IS NOT NULL + AND trip_distance IS NOT NULL + LIMIT 10000000 + """, + use_bqstorage_api=True, + configuration={ + "query": { + "destinationTable": { + "projectId": random_dataset.project, + "datasetId": random_dataset.dataset_id, + "tableId": "".join( + [ + "test_read_gbq_w_bqstorage_api_", + str(uuid.uuid4()).replace("-", "_"), + ] + ), + }, + "writeDisposition": "WRITE_TRUNCATE", + } + }, + ) + assert len(df) == 10000000 diff --git a/packages/pandas-gbq/tests/unit/test_timestamp.py b/packages/pandas-gbq/tests/unit/test_timestamp.py new file mode 100644 index 000000000000..cdd0c55e0f26 --- /dev/null +++ b/packages/pandas-gbq/tests/unit/test_timestamp.py @@ -0,0 +1,72 @@ +"""Unit tests for TIMESTAMP data type helpers.""" + +import pandas +import pandas.testing + +import pytest + + +@pytest.fixture +def module_under_test(): + from pandas_gbq import timestamp + + return timestamp + + +def test_localize_df_with_empty_dataframe(module_under_test): + df = pandas.DataFrame({"timestamp_col": [], "other_col": []}) + original = df.copy() + bq_schema = [ + {"name": "timestamp_col", "type": "TIMESTAMP"}, + {"name": "other_col", "type": "STRING"}, + ] + + localized = module_under_test.localize_df(df, bq_schema) + + # Empty DataFrames should be unchanged. + assert localized is df + pandas.testing.assert_frame_equal(localized, original) + + +def test_localize_df_with_no_timestamp_columns(module_under_test): + df = pandas.DataFrame( + {"integer_col": [1, 2, 3], "float_col": [0.1, 0.2, 0.3]} + ) + original = df.copy() + bq_schema = [ + {"name": "integer_col", "type": "INTEGER"}, + {"name": "float_col", "type": "FLOAT"}, + ] + + localized = module_under_test.localize_df(df, bq_schema) + + # DataFrames with no TIMESTAMP columns should be unchanged. + assert localized is df + pandas.testing.assert_frame_equal(localized, original) + + +def test_localize_df_with_timestamp_column(module_under_test): + df = pandas.DataFrame( + { + "integer_col": [1, 2, 3], + "timestamp_col": pandas.Series( + [ + "2011-01-01 01:02:03", + "2012-02-02 04:05:06", + "2013-03-03 07:08:09", + ], + dtype="datetime64[ns]", + ), + "float_col": [0.1, 0.2, 0.3], + } + ) + expected = df.copy() + expected["timestamp_col"] = df["timestamp_col"].dt.tz_localize("UTC") + bq_schema = [ + {"name": "integer_col", "type": "INTEGER"}, + {"name": "timestamp_col", "type": "TIMESTAMP"}, + {"name": "float_col", "type": "FLOAT"}, + ] + + localized = module_under_test.localize_df(df, bq_schema) + pandas.testing.assert_frame_equal(localized, expected) From 3be0be5d8e9c2fe14754c2bf6e2b2611afd684b3 Mon Sep 17 00:00:00 2001 From: Nico Albers Date: Mon, 24 Feb 2020 15:11:51 +0100 Subject: [PATCH 204/519] ENH: restrict to Python 3.5 and higher in setup.py (#314) --- packages/pandas-gbq/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index 0b97bbfd5315..14a49bac7f29 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -52,6 +52,7 @@ def readme(): keywords="data", install_requires=INSTALL_REQUIRES, extras_require=extras, + python_requires=">=3.5", packages=find_packages(exclude=["contrib", "docs", "tests*"]), test_suite="tests", ) From aae55144a9413162fbd854f45410a5c3eee9f461 Mon Sep 17 00:00:00 2001 From: shan Date: Wed, 29 Apr 2020 22:20:32 +0200 Subject: [PATCH 205/519] BUG: use original schema when appending (#318) * BUG: use original schema when appending Don't overwrite table schema when appending to an existing table * python 3.5 doesn't support f-string * cln: refactor `to_gbq` to avoid unnecessary extra table GET HTTP calls pandas-gbq already gets the table metadata when checking if a table exists. This refactoring avoids extra calls to get the table metadata when checking the schema. also, fix a bug where update_schema appends columns that aren't in the dataframe to the schema sent in the API request * doc: add fix to changelog * doc: revert accidental whitespace change Co-authored-by: Tim Swast --- .../.github/PULL_REQUEST_TEMPLATE.md | 4 + packages/pandas-gbq/docs/source/changelog.rst | 8 ++ packages/pandas-gbq/pandas_gbq/gbq.py | 92 +++++++------------ packages/pandas-gbq/pandas_gbq/schema.py | 56 ++++++++++- packages/pandas-gbq/tests/system/test_gbq.py | 68 ++++++-------- packages/pandas-gbq/tests/unit/test_schema.py | 63 ++++++++++--- 6 files changed, 180 insertions(+), 111 deletions(-) create mode 100644 packages/pandas-gbq/.github/PULL_REQUEST_TEMPLATE.md diff --git a/packages/pandas-gbq/.github/PULL_REQUEST_TEMPLATE.md b/packages/pandas-gbq/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000000..e434e5ea3faf --- /dev/null +++ b/packages/pandas-gbq/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,4 @@ +- [ ] closes #xxxx +- [ ] tests added / passed +- [ ] passes `nox -s blacken lint` +- [ ] `docs/source/changelog.rst` entry \ No newline at end of file diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 56586562029d..dc0867d7e421 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,6 +1,14 @@ Changelog ========= +.. _changelog-0.13.2: + +0.13.2 / TBD +------------ + +- Fix ``Provided Schema does not match Table`` error when the existing table + contains required fields. (:issue:`315`) + .. _changelog-0.13.1: 0.13.1 / 2020-02-13 diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index db699096c058..6cd5d739a365 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -5,6 +5,15 @@ import numpy as np +# Required dependencies, but treat as optional so that _test_google_api_imports +# can provide a better error message. +try: + from google.api_core import exceptions as google_exceptions + from google.cloud import bigquery +except ImportError: # pragma: NO COVER + bigquery = None + google_exceptions = None + try: # The BigQuery Storage API client is an optional dependency. It is only # required when use_bqstorage_api=True. @@ -388,7 +397,6 @@ def sizeof_fmt(num, suffix="B"): return fmt % (num, "Y", suffix) def get_client(self): - from google.cloud import bigquery import pandas try: @@ -429,7 +437,6 @@ def run_query( ): from concurrent.futures import TimeoutError from google.auth.exceptions import RefreshError - from google.cloud import bigquery job_config = { "query": { @@ -640,15 +647,6 @@ def schema(self, dataset_id, table_id): except self.http_error as ex: self.process_http_error(ex) - def _clean_schema_fields(self, fields): - """Return a sanitized version of the schema for comparisons.""" - fields_sorted = sorted(fields, key=lambda field: field["name"]) - # Ignore mode and description when comparing schemas. - return [ - {"name": field["name"], "type": field["type"]} - for field in fields_sorted - ] - def verify_schema(self, dataset_id, table_id, schema): """Indicate whether schemas match exactly @@ -672,43 +670,12 @@ def verify_schema(self, dataset_id, table_id, schema): Whether the schemas match """ - fields_remote = self._clean_schema_fields( + fields_remote = pandas_gbq.schema._clean_schema_fields( self.schema(dataset_id, table_id) ) - fields_local = self._clean_schema_fields(schema["fields"]) - + fields_local = pandas_gbq.schema._clean_schema_fields(schema["fields"]) return fields_remote == fields_local - def schema_is_subset(self, dataset_id, table_id, schema): - """Indicate whether the schema to be uploaded is a subset - - Compare the BigQuery table identified in the parameters with - the schema passed in and indicate whether a subset of the fields in - the former are present in the latter. Order is not considered. - - Parameters - ---------- - dataset_id : str - Name of the BigQuery dataset for the table - table_id : str - Name of the BigQuery table - schema : list(dict) - Schema for comparison. Each item should have - a 'name' and a 'type' - - Returns - ------- - bool - Whether the passed schema is a subset - """ - - fields_remote = self._clean_schema_fields( - self.schema(dataset_id, table_id) - ) - fields_local = self._clean_schema_fields(schema["fields"]) - - return all(field in fields_remote for field in fields_local) - def delete_and_recreate_table(self, dataset_id, table_id, table_schema): table = _Table( self.project_id, dataset_id, credentials=self.credentials @@ -1141,7 +1108,6 @@ def to_gbq( """ _test_google_api_imports() - from pandas_gbq import schema if verbose is not None and SHOW_VERBOSE_DEPRECATION: warnings.warn( @@ -1168,25 +1134,31 @@ def to_gbq( credentials=credentials, private_key=private_key, ) + bqclient = connector.client dataset_id, table_id = destination_table.rsplit(".", 1) - table = _Table( - project_id, - dataset_id, - location=location, - credentials=connector.credentials, - ) - default_schema = _generate_bq_schema(dataframe) if not table_schema: table_schema = default_schema else: - table_schema = schema.update_schema( + table_schema = pandas_gbq.schema.update_schema( default_schema, dict(fields=table_schema) ) # If table exists, check if_exists parameter - if table.exists(table_id): + try: + table = bqclient.get_table(destination_table) + except google_exceptions.NotFound: + table_connector = _Table( + project_id, + dataset_id, + location=location, + credentials=connector.credentials, + ) + table_connector.create(table_id, table_schema) + else: + original_schema = pandas_gbq.schema.to_pandas_gbq(table.schema) + if if_exists == "fail": raise TableCreationError( "Could not create the table because it " @@ -1199,16 +1171,20 @@ def to_gbq( dataset_id, table_id, table_schema ) elif if_exists == "append": - if not connector.schema_is_subset( - dataset_id, table_id, table_schema + if not pandas_gbq.schema.schema_is_subset( + original_schema, table_schema ): raise InvalidSchema( "Please verify that the structure and " "data types in the DataFrame match the " "schema of the destination table." ) - else: - table.create(table_id, table_schema) + + # Update the local `table_schema` so mode matches. + # See: https://github.com/pydata/pandas-gbq/issues/315 + table_schema = pandas_gbq.schema.update_schema( + table_schema, original_schema + ) if dataframe.empty: # Create the table (if needed), but don't try to run a load job with an diff --git a/packages/pandas-gbq/pandas_gbq/schema.py b/packages/pandas-gbq/pandas_gbq/schema.py index bb18fabc6a1c..4154044bf74c 100644 --- a/packages/pandas-gbq/pandas_gbq/schema.py +++ b/packages/pandas-gbq/pandas_gbq/schema.py @@ -3,6 +3,59 @@ import copy +def to_pandas_gbq(client_schema): + """Given a sequence of :class:`google.cloud.bigquery.schema.SchemaField`, + return a schema in pandas-gbq API format. + """ + remote_fields = [ + field_remote.to_api_repr() for field_remote in client_schema + ] + for field in remote_fields: + field["type"] = field["type"].upper() + field["mode"] = field["mode"].upper() + + return {"fields": remote_fields} + + +def _clean_schema_fields(fields): + """Return a sanitized version of the schema for comparisons. + + The ``mode`` and ``description`` properties areis ignored because they + are not generated by func:`pandas_gbq.schema.generate_bq_schema`. + """ + fields_sorted = sorted(fields, key=lambda field: field["name"]) + return [ + {"name": field["name"], "type": field["type"]} + for field in fields_sorted + ] + + +def schema_is_subset(schema_remote, schema_local): + """Indicate whether the schema to be uploaded is a subset + + Compare the BigQuery table identified in the parameters with + the schema passed in and indicate whether a subset of the fields in + the former are present in the latter. Order is not considered. + + Parameters + ---------- + schema_remote : dict + Schema for comparison. Each item of ``fields`` should have a 'name' + and a 'type' + schema_local : dict + Schema for comparison. Each item of ``fields`` should have a 'name' + and a 'type' + + Returns + ------- + bool + Whether the passed schema is a subset + """ + fields_remote = _clean_schema_fields(schema_remote.get("fields", [])) + fields_local = _clean_schema_fields(schema_local.get("fields", [])) + return all(field in fields_remote for field in fields_local) + + def generate_bq_schema(dataframe, default_type="STRING"): """Given a passed dataframe, generate the associated Google BigQuery schema. @@ -59,9 +112,6 @@ def update_schema(schema_old, schema_new): if name in field_indices: # replace old field with new field of same name output_fields[field_indices[name]] = field - else: - # add new field - output_fields.append(field) return {"fields": output_fields} diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index 785e47101f13..ebb211a7e290 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -12,6 +12,7 @@ import pytz from pandas_gbq import gbq +import pandas_gbq.schema TABLE_ID = "new_test" @@ -1637,7 +1638,7 @@ def test_retrieve_schema(gbq_table, gbq_connector): } gbq_table.create(table_id, test_schema) - actual = gbq_connector._clean_schema_fields( + actual = pandas_gbq.schema._clean_schema_fields( gbq_connector.schema(gbq_table.dataset_id, table_id) ) expected = [ @@ -1649,48 +1650,39 @@ def test_retrieve_schema(gbq_table, gbq_connector): assert expected == actual, "Expected schema used to create table" -def test_schema_is_subset_passes_if_subset(gbq_table, gbq_connector): - # Issue #24 schema_is_subset indicates whether the schema of the - # dataframe is a subset of the schema of the bigquery table - table_id = "test_schema_is_subset_passes_if_subset" +def test_to_gbq_does_not_override_mode(gbq_table, gbq_connector): + # See: https://github.com/pydata/pandas-gbq/issues/315 + table_id = "test_to_gbq_does_not_override_mode" table_schema = { "fields": [ - {"name": "A", "type": "FLOAT"}, - {"name": "B", "type": "FLOAT"}, - {"name": "C", "type": "STRING"}, - ] - } - tested_schema = { - "fields": [ - {"name": "A", "type": "FLOAT"}, - {"name": "B", "type": "FLOAT"}, + { + "mode": "REQUIRED", + "name": "A", + "type": "FLOAT", + "description": "A", + }, + { + "mode": "NULLABLE", + "name": "B", + "type": "FLOAT", + "description": "B", + }, + { + "mode": "NULLABLE", + "name": "C", + "type": "STRING", + "description": "C", + }, ] } gbq_table.create(table_id, table_schema) - assert gbq_connector.schema_is_subset( - gbq_table.dataset_id, table_id, tested_schema + gbq.to_gbq( + pandas.DataFrame({"A": [1.0], "B": [2.0], "C": ["a"]}), + "{0}.{1}".format(gbq_table.dataset_id, table_id), + project_id=gbq_connector.project_id, + if_exists="append", ) - -def test_schema_is_subset_fails_if_not_subset(gbq_table, gbq_connector): - # For pull request #24 - table_id = "test_schema_is_subset_fails_if_not_subset" - table_schema = { - "fields": [ - {"name": "A", "type": "FLOAT"}, - {"name": "B", "type": "FLOAT"}, - {"name": "C", "type": "STRING"}, - ] - } - tested_schema = { - "fields": [ - {"name": "A", "type": "FLOAT"}, - {"name": "C", "type": "FLOAT"}, - ] - } - - gbq_table.create(table_id, table_schema) - assert not gbq_connector.schema_is_subset( - gbq_table.dataset_id, table_id, tested_schema - ) + actual = gbq_connector.schema(gbq_table.dataset_id, table_id) + assert table_schema["fields"] == actual diff --git a/packages/pandas-gbq/tests/unit/test_schema.py b/packages/pandas-gbq/tests/unit/test_schema.py index af3b204360ab..2b7324281fef 100644 --- a/packages/pandas-gbq/tests/unit/test_schema.py +++ b/packages/pandas-gbq/tests/unit/test_schema.py @@ -3,7 +3,48 @@ import pandas import pytest -import pandas_gbq.schema + +@pytest.fixture +def module_under_test(): + import pandas_gbq.schema + + return pandas_gbq.schema + + +def test_schema_is_subset_passes_if_subset(module_under_test): + # Issue #24 schema_is_subset indicates whether the schema of the + # dataframe is a subset of the schema of the bigquery table + table_schema = { + "fields": [ + {"name": "A", "type": "FLOAT"}, + {"name": "B", "type": "FLOAT"}, + {"name": "C", "type": "STRING"}, + ] + } + tested_schema = { + "fields": [ + {"name": "A", "type": "FLOAT"}, + {"name": "B", "type": "FLOAT"}, + ] + } + assert module_under_test.schema_is_subset(table_schema, tested_schema) + + +def test_schema_is_subset_fails_if_not_subset(module_under_test): + table_schema = { + "fields": [ + {"name": "A", "type": "FLOAT"}, + {"name": "B", "type": "FLOAT"}, + {"name": "C", "type": "STRING"}, + ] + } + tested_schema = { + "fields": [ + {"name": "A", "type": "FLOAT"}, + {"name": "C", "type": "FLOAT"}, + ] + } + assert not module_under_test.schema_is_subset(table_schema, tested_schema) @pytest.mark.parametrize( @@ -51,8 +92,8 @@ ), ], ) -def test_generate_bq_schema(dataframe, expected_schema): - schema = pandas_gbq.schema.generate_bq_schema(dataframe) +def test_generate_bq_schema(module_under_test, dataframe, expected_schema): + schema = module_under_test.generate_bq_schema(dataframe) assert schema == expected_schema @@ -62,16 +103,13 @@ def test_generate_bq_schema(dataframe, expected_schema): ( {"fields": [{"name": "col1", "type": "INTEGER"}]}, {"fields": [{"name": "col2", "type": "TIMESTAMP"}]}, - { - "fields": [ - {"name": "col1", "type": "INTEGER"}, - {"name": "col2", "type": "TIMESTAMP"}, - ] - }, + # Ignore fields that aren't in the DataFrame. + {"fields": [{"name": "col1", "type": "INTEGER"}]}, ), ( {"fields": [{"name": "col1", "type": "INTEGER"}]}, {"fields": [{"name": "col1", "type": "BOOLEAN"}]}, + # Update type for fields that are in the DataFrame. {"fields": [{"name": "col1", "type": "BOOLEAN"}]}, ), ( @@ -91,12 +129,13 @@ def test_generate_bq_schema(dataframe, expected_schema): "fields": [ {"name": "col1", "type": "INTEGER"}, {"name": "col2", "type": "BOOLEAN"}, - {"name": "col3", "type": "FLOAT"}, ] }, ), ], ) -def test_update_schema(schema_old, schema_new, expected_output): - output = pandas_gbq.schema.update_schema(schema_old, schema_new) +def test_update_schema( + module_under_test, schema_old, schema_new, expected_output +): + output = module_under_test.update_schema(schema_old, schema_new) assert output == expected_output From 015c432b1cb0ca3532fb08413a3f5db5798ff3d3 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 14 May 2020 10:40:55 -0500 Subject: [PATCH 206/519] release 0.13.2 --- packages/pandas-gbq/docs/source/changelog.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index dc0867d7e421..c82fd252e4af 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -3,8 +3,8 @@ Changelog .. _changelog-0.13.2: -0.13.2 / TBD ------------- +0.13.2 / 2020-05-14 +------------------- - Fix ``Provided Schema does not match Table`` error when the existing table contains required fields. (:issue:`315`) From 1d1ae6f5b7a11c4bfecb66f283d68934ebfaadc9 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 30 Sep 2020 11:19:03 -0500 Subject: [PATCH 207/519] BUG: include needed "extras" from google-cloud-bigquery package (#330) This way necessary version ranges for transitive dependencies such as google-cloud-bigquery-storage and pyarrow are included in the installation. --- packages/pandas-gbq/docs/source/changelog.rst | 9 +++++++++ packages/pandas-gbq/setup.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index c82fd252e4af..a5754a27a1c6 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,6 +1,15 @@ Changelog ========= +.. _changelog-0.13.3: + +0.13.3 / 2020-09-30 +------------------- + +- Include needed "extras" from ``google-cloud-bigquery`` package as + dependencies. Exclude incompatible 2.0 version. (:issue:`324`, :issue:`329`) + + .. _changelog-0.13.2: 0.13.2 / 2020-05-14 diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index 14a49bac7f29..c4856a26f77e 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -22,7 +22,7 @@ def readme(): "pydata-google-auth", "google-auth", "google-auth-oauthlib", - "google-cloud-bigquery>=1.11.1", + "google-cloud-bigquery[bqstorage,pandas]>=1.11.1,<2.0.0dev", ] extras = {"tqdm": "tqdm>=4.23.0"} From 896e582b6fc1732055e391d6d0ac6e01f2536edb Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 2 Oct 2020 10:34:53 -0500 Subject: [PATCH 208/519] TST: refactor pip tests to use constraints files (#331) * TST: refactor pip tests to use constraints files Realized that tests were failing on Python 3.5 with the specified versions and bumped the minimum pandas version to 0.20.1 (for [pandas.testing](https://stackoverflow.com/a/53631895/101923) support). Added tests to run against Python 3.8 (just in time for Python 3.9 to come out next week!) * update changelog * update circleci config --- packages/pandas-gbq/.circleci/config.yml | 19 +++++++++---- ...quirements-3.5.pip => constraints-3.5.pip} | 2 +- ...quirements-3.6.pip => constraints-3.6.pip} | 0 ...quirements-3.7.pip => constraints-3.7.pip} | 0 packages/pandas-gbq/ci/constraints-3.8.pip | 1 + ...Y.conda => requirements-3.8-NIGHTLY.conda} | 0 packages/pandas-gbq/docs/source/changelog.rst | 19 +++++++++++-- packages/pandas-gbq/noxfile.py | 27 +++++++++++-------- packages/pandas-gbq/setup.py | 3 ++- 9 files changed, 51 insertions(+), 20 deletions(-) rename packages/pandas-gbq/ci/{requirements-3.5.pip => constraints-3.5.pip} (88%) rename packages/pandas-gbq/ci/{requirements-3.6.pip => constraints-3.6.pip} (100%) rename packages/pandas-gbq/ci/{requirements-3.7.pip => constraints-3.7.pip} (100%) create mode 100644 packages/pandas-gbq/ci/constraints-3.8.pip rename packages/pandas-gbq/ci/{requirements-3.7-NIGHTLY.conda => requirements-3.8-NIGHTLY.conda} (100%) diff --git a/packages/pandas-gbq/.circleci/config.yml b/packages/pandas-gbq/.circleci/config.yml index 053280cddf1c..a3863bd44cc2 100644 --- a/packages/pandas-gbq/.circleci/config.yml +++ b/packages/pandas-gbq/.circleci/config.yml @@ -15,7 +15,7 @@ jobs: steps: - checkout - run: ci/config_auth.sh - - run: nox -s unit-3.6 system-3.6 + - run: nox -s unit-3.6 "pip-3.7": docker: @@ -23,7 +23,15 @@ jobs: steps: - checkout - run: ci/config_auth.sh - - run: nox -s unit-3.7 system-3.7 cover + - run: nox -s unit-3.7 + + "pip-3.8": + docker: + - image: thekevjames/nox + steps: + - checkout + - run: ci/config_auth.sh + - run: nox -s unit-3.8 system-3.8 cover "lint": docker: @@ -47,11 +55,11 @@ jobs: - checkout - run: ci/config_auth.sh - run: ci/run_conda.sh - "conda-3.7-NIGHTLY": + "conda-3.8-NIGHTLY": docker: - image: continuumio/miniconda3 environment: - PYTHON: "3.7" + PYTHON: "3.8" PANDAS: "NIGHTLY" steps: - checkout @@ -65,6 +73,7 @@ workflows: - "pip-3.5" - "pip-3.6" - "pip-3.7" + - "pip-3.8" - lint - "conda-3.6-0.20.1" - - "conda-3.7-NIGHTLY" \ No newline at end of file + - "conda-3.8-NIGHTLY" \ No newline at end of file diff --git a/packages/pandas-gbq/ci/requirements-3.5.pip b/packages/pandas-gbq/ci/constraints-3.5.pip similarity index 88% rename from packages/pandas-gbq/ci/requirements-3.5.pip rename to packages/pandas-gbq/ci/constraints-3.5.pip index 2f9911de8262..2fd6f28a708d 100644 --- a/packages/pandas-gbq/ci/requirements-3.5.pip +++ b/packages/pandas-gbq/ci/constraints-3.5.pip @@ -1,5 +1,5 @@ numpy==1.13.3 -pandas==0.19.0 +pandas==0.20.1 google-auth==1.4.1 google-auth-oauthlib==0.0.1 google-cloud-bigquery==1.11.1 diff --git a/packages/pandas-gbq/ci/requirements-3.6.pip b/packages/pandas-gbq/ci/constraints-3.6.pip similarity index 100% rename from packages/pandas-gbq/ci/requirements-3.6.pip rename to packages/pandas-gbq/ci/constraints-3.6.pip diff --git a/packages/pandas-gbq/ci/requirements-3.7.pip b/packages/pandas-gbq/ci/constraints-3.7.pip similarity index 100% rename from packages/pandas-gbq/ci/requirements-3.7.pip rename to packages/pandas-gbq/ci/constraints-3.7.pip diff --git a/packages/pandas-gbq/ci/constraints-3.8.pip b/packages/pandas-gbq/ci/constraints-3.8.pip new file mode 100644 index 000000000000..1411a4a0b5ab --- /dev/null +++ b/packages/pandas-gbq/ci/constraints-3.8.pip @@ -0,0 +1 @@ +pandas \ No newline at end of file diff --git a/packages/pandas-gbq/ci/requirements-3.7-NIGHTLY.conda b/packages/pandas-gbq/ci/requirements-3.8-NIGHTLY.conda similarity index 100% rename from packages/pandas-gbq/ci/requirements-3.7-NIGHTLY.conda rename to packages/pandas-gbq/ci/requirements-3.8-NIGHTLY.conda diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index a5754a27a1c6..a6ff82f55189 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,6 +1,23 @@ Changelog ========= +.. _changelog-0.14.0: + +0.14.0 / TBD +------------ + +Dependency updates +~~~~~~~~~~~~~~~~~~ + +- Update the minimum version of ``pandas`` to 0.20.1. + (:issue:`331`) + +Internal changes +~~~~~~~~~~~~~~~~ + +- Update tests to run against for Python 3.8. (:issue:`331`) + + .. _changelog-0.13.3: 0.13.3 / 2020-09-30 @@ -9,7 +26,6 @@ Changelog - Include needed "extras" from ``google-cloud-bigquery`` package as dependencies. Exclude incompatible 2.0 version. (:issue:`324`, :issue:`329`) - .. _changelog-0.13.2: 0.13.2 / 2020-05-14 @@ -26,7 +42,6 @@ Changelog - Fix ``AttributeError`` with BQ Storage API to download empty results. (:issue:`299`) - .. _changelog-0.13.0: 0.13.0 / 2019-12-12 diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 1ab4ce898544..e30593be0392 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -10,8 +10,9 @@ import nox -supported_pythons = ["3.5", "3.6", "3.7"] -latest_python = "3.7" +supported_pythons = ["3.5", "3.6", "3.7", "3.8"] +system_test_pythons = ["3.5", "3.8"] +latest_python = "3.8" # Use a consistent version of black so CI is deterministic. black_package = "black==19.10b0" @@ -35,7 +36,14 @@ def blacken(session): @nox.session(python=supported_pythons) def unit(session): session.install("pytest", "pytest-cov") - session.install("-e", ".") + session.install( + "-e", + ".", + # Use dependencies versions from constraints file. This enables testing + # across a more full range of versions of the dependencies. + "-c", + os.path.join(".", "ci", "constraints-{}.pip".format(session.python)), + ) session.run( "pytest", os.path.join(".", "tests", "unit"), @@ -77,19 +85,16 @@ def docs(session): ) -@nox.session(python=supported_pythons) +@nox.session(python=system_test_pythons) def system(session): session.install("pytest", "pytest-cov") - session.install( - "-r", - os.path.join(".", "ci", "requirements-{}.pip".format(session.python)), - ) session.install( "-e", ".", - # Use dependencies from requirements file instead. - # This enables testing with specific versions of the dependencies. - "--no-dependencies", + # Use dependencies versions from constraints file. This enables testing + # across a more full range of versions of the dependencies. + "-c", + os.path.join(".", "ci", "constraints-{}.pip".format(session.python)), ) # Skip local auth tests on CI. diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index c4856a26f77e..bf2fde71f26b 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -18,7 +18,7 @@ def readme(): INSTALL_REQUIRES = [ "setuptools", - "pandas>=0.19.0", + "pandas>=0.20.1", "pydata-google-auth", "google-auth", "google-auth-oauthlib", @@ -47,6 +47,7 @@ def readme(): "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", "Topic :: Scientific/Engineering", ], keywords="data", From e28767695bf0730b670ae1165f98b49d3cb57651 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 2 Oct 2020 13:09:23 -0500 Subject: [PATCH 209/519] ENH: add dtypes argument to read_gbq (#333) Use this argument to override the default ``dtype`` for a particular column in the query results. For example, this can be used to select nullable integer columns as the ``Int64`` nullable integer pandas extension type. df = gbq.read_gbq( "SELECT CAST(NULL AS INT64) AS null_integer", dtypes={"null_integer": "Int64"}, ) --- packages/pandas-gbq/docs/source/changelog.rst | 15 +++- packages/pandas-gbq/pandas_gbq/gbq.py | 22 +++++- packages/pandas-gbq/tests/system/test_gbq.py | 73 ++++++++++++++----- packages/pandas-gbq/tests/unit/test_gbq.py | 21 ++++++ 4 files changed, 108 insertions(+), 23 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index a6ff82f55189..f1ebae903765 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -6,6 +6,19 @@ Changelog 0.14.0 / TBD ------------ +- Add ``dtypes`` argument to ``read_gbq``. Use this argument to override the + default ``dtype`` for a particular column in the query results. For + example, this can be used to select nullable integer columns as the + ``Int64`` nullable integer pandas extension type. (:issue:`242`, + :issue:`332`) + +.. code-block:: python + + df = gbq.read_gbq( + "SELECT CAST(NULL AS INT64) AS null_integer", + dtypes={"null_integer": "Int64"}, + ) + Dependency updates ~~~~~~~~~~~~~~~~~~ @@ -15,7 +28,7 @@ Dependency updates Internal changes ~~~~~~~~~~~~~~~~ -- Update tests to run against for Python 3.8. (:issue:`331`) +- Update tests to run against Python 3.8. (:issue:`331`) .. _changelog-0.13.3: diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 6cd5d739a365..4671227dbd33 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -526,19 +526,28 @@ def run_query( ) ) + dtypes = kwargs.get("dtypes") return self._download_results( query_reply, max_results=max_results, progress_bar_type=progress_bar_type, + user_dtypes=dtypes, ) def _download_results( - self, query_job, max_results=None, progress_bar_type=None + self, + query_job, + max_results=None, + progress_bar_type=None, + user_dtypes=None, ): # No results are desired, so don't bother downloading anything. if max_results == 0: return None + if user_dtypes is None: + user_dtypes = {} + try: bqstorage_client = None if max_results is None: @@ -555,9 +564,10 @@ def _download_results( ) schema_fields = [field.to_api_repr() for field in rows_iter.schema] - nullsafe_dtypes = _bqschema_to_nullsafe_dtypes(schema_fields) + conversion_dtypes = _bqschema_to_nullsafe_dtypes(schema_fields) + conversion_dtypes.update(user_dtypes) df = rows_iter.to_dataframe( - dtypes=nullsafe_dtypes, + dtypes=conversion_dtypes, bqstorage_client=bqstorage_client, progress_bar_type=progress_bar_type, ) @@ -790,6 +800,7 @@ def read_gbq( verbose=None, private_key=None, progress_bar_type="tqdm", + dtypes=None, ): r"""Load data from Google BigQuery using google-cloud-python @@ -910,6 +921,10 @@ def read_gbq( ``'tqdm_gui'`` Use the :func:`tqdm.tqdm_gui` function to display a progress bar as a graphical dialog box. + dtypes : dict, optional + A dictionary of column names to pandas ``dtype``. The provided + ``dtype`` is used when constructing the series for the column + specified. Otherwise, a default ``dtype`` is used. verbose : None, deprecated Deprecated in Pandas-GBQ 0.4.0. Use the `logging module to adjust verbosity instead @@ -965,6 +980,7 @@ def read_gbq( configuration=configuration, max_results=max_results, progress_bar_type=progress_bar_type, + dtypes=dtypes, ) # Reindex the DataFrame on the provided column diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index ebb211a7e290..c583b48baddc 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -8,6 +8,11 @@ import pandas.api.types import pandas.util.testing as tm from pandas import DataFrame, NaT + +try: + import pkg_resources # noqa +except ImportError: + raise ImportError("Could not import pkg_resources (setuptools).") import pytest import pytz @@ -16,14 +21,14 @@ TABLE_ID = "new_test" +PANDAS_VERSION = pkg_resources.parse_version(pandas.__version__) +NULLABLE_INT_PANDAS_VERSION = pkg_resources.parse_version("0.24.0") +NULLABLE_INT_MESSAGE = ( + "Require pandas 0.24+ in order to use nullable integer type." +) def test_imports(): - try: - import pkg_resources # noqa - except ImportError: - raise ImportError("Could not import pkg_resources (setuptools).") - gbq._test_google_api_imports() @@ -87,26 +92,39 @@ def test_should_properly_handle_null_strings(self, project_id): tm.assert_frame_equal(df, DataFrame({"null_string": [None]})) def test_should_properly_handle_valid_integers(self, project_id): - query = "SELECT INTEGER(3) AS valid_integer" + query = "SELECT CAST(3 AS INT64) AS valid_integer" df = gbq.read_gbq( query, project_id=project_id, credentials=self.credentials, - dialect="legacy", + dialect="standard", ) tm.assert_frame_equal(df, DataFrame({"valid_integer": [3]})) def test_should_properly_handle_nullable_integers(self, project_id): + if PANDAS_VERSION < NULLABLE_INT_PANDAS_VERSION: + pytest.skip(msg=NULLABLE_INT_MESSAGE) + query = """SELECT * FROM - (SELECT 1 AS nullable_integer), - (SELECT NULL AS nullable_integer)""" + UNNEST([1, NULL]) AS nullable_integer + """ df = gbq.read_gbq( query, project_id=project_id, credentials=self.credentials, - dialect="legacy", + dialect="standard", + dtypes={"nullable_integer": "Int64"}, + ) + tm.assert_frame_equal( + df, + DataFrame( + { + "nullable_integer": pandas.Series( + [1, pandas.NA], dtype="Int64" + ) + } + ), ) - tm.assert_frame_equal(df, DataFrame({"nullable_integer": [1, None]})) def test_should_properly_handle_valid_longs(self, project_id): query = "SELECT 1 << 62 AS valid_long" @@ -114,35 +132,52 @@ def test_should_properly_handle_valid_longs(self, project_id): query, project_id=project_id, credentials=self.credentials, - dialect="legacy", + dialect="standard", ) tm.assert_frame_equal(df, DataFrame({"valid_long": [1 << 62]})) def test_should_properly_handle_nullable_longs(self, project_id): + if PANDAS_VERSION < NULLABLE_INT_PANDAS_VERSION: + pytest.skip(msg=NULLABLE_INT_MESSAGE) + query = """SELECT * FROM - (SELECT 1 << 62 AS nullable_long), - (SELECT NULL AS nullable_long)""" + UNNEST([1 << 62, NULL]) AS nullable_long + """ df = gbq.read_gbq( query, project_id=project_id, credentials=self.credentials, - dialect="legacy", + dialect="standard", + dtypes={"nullable_long": "Int64"}, ) tm.assert_frame_equal( - df, DataFrame({"nullable_long": [1 << 62, None]}) + df, + DataFrame( + { + "nullable_long": pandas.Series( + [1 << 62, pandas.NA], dtype="Int64" + ) + } + ), ) def test_should_properly_handle_null_integers(self, project_id): - query = "SELECT INTEGER(NULL) AS null_integer" + if PANDAS_VERSION < NULLABLE_INT_PANDAS_VERSION: + pytest.skip(msg=NULLABLE_INT_MESSAGE) + + query = "SELECT CAST(NULL AS INT64) AS null_integer" df = gbq.read_gbq( query, project_id=project_id, credentials=self.credentials, - dialect="legacy", + dialect="standard", + dtypes={"null_integer": "Int64"}, ) tm.assert_frame_equal( df, - DataFrame({"null_integer": pandas.Series([None], dtype="object")}), + DataFrame( + {"null_integer": pandas.Series([pandas.NA], dtype="Int64")} + ), ) def test_should_properly_handle_valid_floats(self, project_id): diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index e0d0c8a407d2..be965c2719cb 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -476,6 +476,27 @@ def test_load_does_not_modify_schema_arg(mock_bigquery_client): assert original_schema == original_schema_cp +def test_read_gbq_passes_dtypes( + mock_bigquery_client, mock_service_account_credentials +): + mock_service_account_credentials.project_id = "service_account_project_id" + df = gbq.read_gbq( + "SELECT 1 AS int_col", + dialect="standard", + credentials=mock_service_account_credentials, + dtypes={"int_col": "my-custom-dtype"}, + ) + assert df is not None + + mock_list_rows = mock_bigquery_client.list_rows("dest", max_results=100) + + mock_list_rows.to_dataframe.assert_called_once_with( + dtypes={"int_col": "my-custom-dtype"}, + bqstorage_client=mock.ANY, + progress_bar_type=mock.ANY, + ) + + def test_read_gbq_calls_tqdm( mock_bigquery_client, mock_service_account_credentials ): From 70a0f018b48c8d9f7a1639ce84f078b65724f6e7 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 5 Oct 2020 15:36:35 -0500 Subject: [PATCH 210/519] BUG: update library to support google-cloud-bigquery 2.0 (#334) * BUG: update library to support google-cloud-bigquery 2.0 Removes references to BQ Storage API beta endpoint. * fix unit test mocks * add date to changelog --- packages/pandas-gbq/conftest.py | 5 +- packages/pandas-gbq/docs/source/changelog.rst | 5 +- packages/pandas-gbq/pandas_gbq/exceptions.py | 9 +++ packages/pandas-gbq/pandas_gbq/gbq.py | 78 +++++++++---------- packages/pandas-gbq/setup.py | 2 +- packages/pandas-gbq/tests/system/conftest.py | 8 +- .../system/test_read_gbq_with_bqstorage.py | 2 +- packages/pandas-gbq/tests/unit/test_gbq.py | 32 ++++++-- 8 files changed, 85 insertions(+), 56 deletions(-) diff --git a/packages/pandas-gbq/conftest.py b/packages/pandas-gbq/conftest.py index 1485e41eb4b9..7f9a67215f55 100644 --- a/packages/pandas-gbq/conftest.py +++ b/packages/pandas-gbq/conftest.py @@ -56,9 +56,12 @@ def bigquery_client(project_id, private_key_path): @pytest.fixture() def random_dataset_id(bigquery_client): import google.api_core.exceptions + from google.cloud import bigquery dataset_id = "".join(["pandas_gbq_", str(uuid.uuid4()).replace("-", "_")]) - dataset_ref = bigquery_client.dataset(dataset_id) + dataset_ref = bigquery.DatasetReference( + bigquery_client.project, dataset_id + ) yield dataset_id try: bigquery_client.delete_dataset(dataset_ref, delete_contents=True) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index f1ebae903765..46570643c886 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -3,8 +3,8 @@ Changelog .. _changelog-0.14.0: -0.14.0 / TBD ------------- +0.14.0 / 2020-10-05 +------------------- - Add ``dtypes`` argument to ``read_gbq``. Use this argument to override the default ``dtype`` for a particular column in the query results. For @@ -22,6 +22,7 @@ Changelog Dependency updates ~~~~~~~~~~~~~~~~~~ +- Support ``google-cloud-bigquery-storage`` 2.0 and higher. (:issue:`329`) - Update the minimum version of ``pandas`` to 0.20.1. (:issue:`331`) diff --git a/packages/pandas-gbq/pandas_gbq/exceptions.py b/packages/pandas-gbq/pandas_gbq/exceptions.py index 96711455fe46..dde45081bc8f 100644 --- a/packages/pandas-gbq/pandas_gbq/exceptions.py +++ b/packages/pandas-gbq/pandas_gbq/exceptions.py @@ -12,3 +12,12 @@ class InvalidPrivateKeyFormat(ValueError): """ pass + + +class PerformanceWarning(RuntimeWarning): + """ + Raised when a performance-related feature is requested, but unsupported. + + Such warnings can occur when dependencies for the requested feature + aren't up-to-date. + """ diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 4671227dbd33..7ae4fcf1c39f 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -14,14 +14,8 @@ bigquery = None google_exceptions = None -try: - # The BigQuery Storage API client is an optional dependency. It is only - # required when use_bqstorage_api=True. - from google.cloud import bigquery_storage_v1beta1 -except ImportError: # pragma: NO COVER - bigquery_storage_v1beta1 = None - from pandas_gbq.exceptions import AccessDenied +from pandas_gbq.exceptions import PerformanceWarning import pandas_gbq.schema import pandas_gbq.timestamp @@ -30,7 +24,9 @@ BIGQUERY_INSTALLED_VERSION = None BIGQUERY_CLIENT_INFO_VERSION = "1.12.0" +BIGQUERY_BQSTORAGE_VERSION = "1.24.0" HAS_CLIENT_INFO = False +HAS_BQSTORAGE_SUPPORT = False try: import tqdm # noqa @@ -39,7 +35,7 @@ def _check_google_client_version(): - global BIGQUERY_INSTALLED_VERSION, HAS_CLIENT_INFO, SHOW_VERBOSE_DEPRECATION + global BIGQUERY_INSTALLED_VERSION, HAS_CLIENT_INFO, HAS_BQSTORAGE_SUPPORT, SHOW_VERBOSE_DEPRECATION try: import pkg_resources @@ -47,11 +43,14 @@ def _check_google_client_version(): except ImportError: raise ImportError("Could not import pkg_resources (setuptools).") - # https://github.com/GoogleCloudPlatform/google-cloud-python/blob/master/bigquery/CHANGELOG.md + # https://github.com/googleapis/python-bigquery/blob/master/CHANGELOG.md bigquery_minimum_version = pkg_resources.parse_version("1.11.0") bigquery_client_info_version = pkg_resources.parse_version( BIGQUERY_CLIENT_INFO_VERSION ) + bigquery_bqstorage_version = pkg_resources.parse_version( + BIGQUERY_BQSTORAGE_VERSION + ) BIGQUERY_INSTALLED_VERSION = pkg_resources.get_distribution( "google-cloud-bigquery" ).parsed_version @@ -59,6 +58,9 @@ def _check_google_client_version(): HAS_CLIENT_INFO = ( BIGQUERY_INSTALLED_VERSION >= bigquery_client_info_version ) + HAS_BQSTORAGE_SUPPORT = ( + BIGQUERY_INSTALLED_VERSION >= bigquery_bqstorage_version + ) if BIGQUERY_INSTALLED_VERSION < bigquery_minimum_version: raise ImportError( @@ -548,14 +550,30 @@ def _download_results( if user_dtypes is None: user_dtypes = {} - try: - bqstorage_client = None - if max_results is None: - # Only use the BigQuery Storage API if the full result set is requested. - bqstorage_client = _make_bqstorage_client( - self.use_bqstorage_api, self.credentials - ) + if self.use_bqstorage_api and not HAS_BQSTORAGE_SUPPORT: + warnings.warn( + ( + "use_bqstorage_api was set, but have google-cloud-bigquery " + "version {}. Requires google-cloud-bigquery version " + "{} or later." + ).format( + BIGQUERY_INSTALLED_VERSION, BIGQUERY_BQSTORAGE_VERSION + ), + PerformanceWarning, + stacklevel=4, + ) + + create_bqstorage_client = self.use_bqstorage_api + if max_results is not None: + create_bqstorage_client = False + + to_dataframe_kwargs = {} + if HAS_BQSTORAGE_SUPPORT: + to_dataframe_kwargs[ + "create_bqstorage_client" + ] = create_bqstorage_client + try: query_job.result() # Get the table schema, so that we can list rows. destination = self.client.get_table(query_job.destination) @@ -568,16 +586,11 @@ def _download_results( conversion_dtypes.update(user_dtypes) df = rows_iter.to_dataframe( dtypes=conversion_dtypes, - bqstorage_client=bqstorage_client, progress_bar_type=progress_bar_type, + **to_dataframe_kwargs ) except self.http_error as ex: self.process_http_error(ex) - finally: - if bqstorage_client: - # Clean up open socket resources. See: - # https://github.com/pydata/pandas-gbq/issues/294 - bqstorage_client.transport.channel.close() if df.empty: df = _cast_empty_df_dtypes(schema_fields, df) @@ -763,27 +776,6 @@ def _cast_empty_df_dtypes(schema_fields, df): return df -def _make_bqstorage_client(use_bqstorage_api, credentials): - if not use_bqstorage_api: - return None - - if bigquery_storage_v1beta1 is None: - raise ImportError( - "Install the google-cloud-bigquery-storage and fastavro/pyarrow " - "packages to use the BigQuery Storage API." - ) - - import google.api_core.gapic_v1.client_info - import pandas - - client_info = google.api_core.gapic_v1.client_info.ClientInfo( - user_agent="pandas-{}".format(pandas.__version__) - ) - return bigquery_storage_v1beta1.BigQueryStorageClient( - credentials=credentials, client_info=client_info - ) - - def read_gbq( query, project_id=None, diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index bf2fde71f26b..4e3d01c9b5dc 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -22,7 +22,7 @@ def readme(): "pydata-google-auth", "google-auth", "google-auth-oauthlib", - "google-cloud-bigquery[bqstorage,pandas]>=1.11.1,<2.0.0dev", + "google-cloud-bigquery[bqstorage,pandas]>=1.11.1,<3.0.0dev", ] extras = {"tqdm": "tqdm>=4.23.0"} diff --git a/packages/pandas-gbq/tests/system/conftest.py b/packages/pandas-gbq/tests/system/conftest.py index 2004519ac97d..9d160b47aed0 100644 --- a/packages/pandas-gbq/tests/system/conftest.py +++ b/packages/pandas-gbq/tests/system/conftest.py @@ -28,7 +28,9 @@ def gbq_connector(project, credentials): def random_dataset(bigquery_client, random_dataset_id): from google.cloud import bigquery - dataset_ref = bigquery_client.dataset(random_dataset_id) + dataset_ref = bigquery.DatasetReference( + bigquery_client.project, random_dataset_id + ) dataset = bigquery.Dataset(dataset_ref) bigquery_client.create_dataset(dataset) return dataset @@ -38,7 +40,9 @@ def random_dataset(bigquery_client, random_dataset_id): def tokyo_dataset(bigquery_client, random_dataset_id): from google.cloud import bigquery - dataset_ref = bigquery_client.dataset(random_dataset_id) + dataset_ref = bigquery.DatasetReference( + bigquery_client.project, random_dataset_id + ) dataset = bigquery.Dataset(dataset_ref) dataset.location = "asia-northeast1" bigquery_client.create_dataset(dataset) diff --git a/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py b/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py index 31f2e48412d1..4dc1fb25ae87 100644 --- a/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py +++ b/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py @@ -6,7 +6,7 @@ import pytest -pytest.importorskip("google.cloud.bigquery_storage_v1beta1") +pytest.importorskip("google.cloud.bigquery", minversion="1.24.0") @pytest.fixture diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index be965c2719cb..1307babe30d9 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -13,7 +13,7 @@ from pandas_gbq import gbq -pytestmark = pytest.mark.filter_warnings( +pytestmark = pytest.mark.filterwarnings( "ignore:credentials from Google Cloud SDK" ) pandas_installed_version = pkg_resources.get_distribution( @@ -490,9 +490,30 @@ def test_read_gbq_passes_dtypes( mock_list_rows = mock_bigquery_client.list_rows("dest", max_results=100) + _, to_dataframe_kwargs = mock_list_rows.to_dataframe.call_args + assert to_dataframe_kwargs["dtypes"] == {"int_col": "my-custom-dtype"} + + +def test_read_gbq_use_bqstorage_api( + mock_bigquery_client, mock_service_account_credentials +): + gbq._check_google_client_version() + if not gbq.HAS_BQSTORAGE_SUPPORT: + pytest.skip("requires BigQuery Storage API") + + mock_service_account_credentials.project_id = "service_account_project_id" + df = gbq.read_gbq( + "SELECT 1 AS int_col", + dialect="standard", + credentials=mock_service_account_credentials, + use_bqstorage_api=True, + ) + assert df is not None + + mock_list_rows = mock_bigquery_client.list_rows("dest", max_results=100) mock_list_rows.to_dataframe.assert_called_once_with( - dtypes={"int_col": "my-custom-dtype"}, - bqstorage_client=mock.ANY, + create_bqstorage_client=True, + dtypes=mock.ANY, progress_bar_type=mock.ANY, ) @@ -511,6 +532,5 @@ def test_read_gbq_calls_tqdm( mock_list_rows = mock_bigquery_client.list_rows("dest", max_results=100) - mock_list_rows.to_dataframe.assert_called_once_with( - dtypes=mock.ANY, bqstorage_client=mock.ANY, progress_bar_type="foobar" - ) + _, to_dataframe_kwargs = mock_list_rows.to_dataframe.call_args + assert to_dataframe_kwargs["progress_bar_type"] == "foobar" From 4223237c9f44956748000690c17553811419ca3f Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 6 Nov 2020 14:26:09 -0600 Subject: [PATCH 211/519] CLN: blacken with same version as Stickler --- packages/pandas-gbq/noxfile.py | 3 ++- packages/pandas-gbq/pandas_gbq/auth.py | 4 +++- packages/pandas-gbq/pandas_gbq/gbq.py | 12 ++++++------ .../tests/system/test_read_gbq_with_bqstorage.py | 5 ++++- packages/pandas-gbq/tests/unit/test_gbq.py | 15 ++++++++------- packages/pandas-gbq/tests/unit/test_load.py | 3 +-- 6 files changed, 24 insertions(+), 18 deletions(-) diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index e30593be0392..6ed0cef1cbe9 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -15,7 +15,8 @@ latest_python = "3.8" # Use a consistent version of black so CI is deterministic. -black_package = "black==19.10b0" +# Should match Stickler: https://stickler-ci.com/docs#black +black_package = "black==20.8b1" @nox.session diff --git a/packages/pandas-gbq/pandas_gbq/auth.py b/packages/pandas-gbq/pandas_gbq/auth.py index 448367b42b8c..27ecc861fc87 100644 --- a/packages/pandas-gbq/pandas_gbq/auth.py +++ b/packages/pandas-gbq/pandas_gbq/auth.py @@ -51,7 +51,9 @@ def get_credentials( return credentials, project_id -def get_credentials_cache(reauth,): +def get_credentials_cache( + reauth, +): import pydata_google_auth.cache if reauth: diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 7ae4fcf1c39f..1b71a7a38cfe 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -1238,7 +1238,7 @@ def _generate_bq_schema(df, default_type="STRING"): issues in the default schema generation. Now that individual columns can be overridden: https://github.com/pydata/pandas-gbq/issues/218, this method can be removed after there is time to migrate away from this - method. """ + method.""" from pandas_gbq import schema return schema.generate_bq_schema(df, default_type=default_type) @@ -1264,7 +1264,7 @@ def __init__( ) def exists(self, table_id): - """ Check if a table exists in Google BigQuery + """Check if a table exists in Google BigQuery Parameters ---------- @@ -1288,7 +1288,7 @@ def exists(self, table_id): self.process_http_error(ex) def create(self, table_id, schema): - """ Create a table in Google BigQuery given a table and schema + """Create a table in Google BigQuery given a table and schema Parameters ---------- @@ -1330,7 +1330,7 @@ def create(self, table_id, schema): self.process_http_error(ex) def delete(self, table_id): - """ Delete a table in Google BigQuery + """Delete a table in Google BigQuery Parameters ---------- @@ -1370,7 +1370,7 @@ def __init__( ) def exists(self, dataset_id): - """ Check if a dataset exists in Google BigQuery + """Check if a dataset exists in Google BigQuery Parameters ---------- @@ -1393,7 +1393,7 @@ def exists(self, dataset_id): self.process_http_error(ex) def create(self, dataset_id): - """ Create a dataset in Google BigQuery + """Create a dataset in Google BigQuery Parameters ---------- diff --git a/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py b/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py index 4dc1fb25ae87..999667919c31 100644 --- a/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py +++ b/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py @@ -30,7 +30,10 @@ def test_empty_results(method_under_test, query_string): See: https://github.com/pydata/pandas-gbq/issues/299 """ - df = method_under_test(query_string, use_bqstorage_api=True,) + df = method_under_test( + query_string, + use_bqstorage_api=True, + ) assert len(df.index) == 0 diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index 1307babe30d9..d2a41608b2ce 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -248,8 +248,8 @@ def test_to_gbq_doesnt_run_query( def test_to_gbq_w_empty_df(mock_bigquery_client): import google.api_core.exceptions - mock_bigquery_client.get_table.side_effect = google.api_core.exceptions.NotFound( - "my_table" + mock_bigquery_client.get_table.side_effect = ( + google.api_core.exceptions.NotFound("my_table") ) gbq.to_gbq(DataFrame(), "my_dataset.my_table", project_id="1234") mock_bigquery_client.create_table.assert_called_with(mock.ANY) @@ -260,11 +260,11 @@ def test_to_gbq_w_empty_df(mock_bigquery_client): def test_to_gbq_creates_dataset(mock_bigquery_client): import google.api_core.exceptions - mock_bigquery_client.get_table.side_effect = google.api_core.exceptions.NotFound( - "my_table" + mock_bigquery_client.get_table.side_effect = ( + google.api_core.exceptions.NotFound("my_table") ) - mock_bigquery_client.get_dataset.side_effect = google.api_core.exceptions.NotFound( - "my_dataset" + mock_bigquery_client.get_dataset.side_effect = ( + google.api_core.exceptions.NotFound("my_dataset") ) gbq.to_gbq(DataFrame([[1]]), "my_dataset.my_table", project_id="1234") mock_bigquery_client.create_dataset.assert_called_with(mock.ANY) @@ -375,7 +375,8 @@ def test_read_gbq_with_old_bq_raises_importerror(): ) as mock_version: mock_version.side_effect = [bigquery_version] gbq.read_gbq( - "SELECT 1", project_id="my-project", + "SELECT 1", + project_id="my-project", ) diff --git a/packages/pandas-gbq/tests/unit/test_load.py b/packages/pandas-gbq/tests/unit/test_load.py index d2b5860e8246..9be8fe89fcaf 100644 --- a/packages/pandas-gbq/tests/unit/test_load.py +++ b/packages/pandas-gbq/tests/unit/test_load.py @@ -38,8 +38,7 @@ def test_encode_chunk_with_floats(): def test_encode_chunk_with_newlines(): - """See: https://github.com/pydata/pandas-gbq/issues/180 - """ + """See: https://github.com/pydata/pandas-gbq/issues/180""" df = pandas.DataFrame({"s": ["abcd", "ef\ngh", "ij\r\nkl"]}) csv_buffer = load.encode_chunk(df) csv_bytes = csv_buffer.read() From 6759941cf3578effe254c0a42ca52bbaa1886946 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 9 Nov 2020 09:30:16 -0600 Subject: [PATCH 212/519] BUG: use greater precision when serializing floating points (#336) * BUG: use greater precision when serializing floating points This allows the exact binary representation to be transferred correctly, round-trip. * blacken * remove f-string * adjust string formatting --- packages/pandas-gbq/conftest.py | 2 +- packages/pandas-gbq/docs/source/changelog.rst | 11 +++++ packages/pandas-gbq/pandas_gbq/load.py | 2 +- .../pandas-gbq/tests/system/test_to_gbq.py | 49 +++++++++++++++++++ packages/pandas-gbq/tests/unit/test_load.py | 32 +++++++++--- 5 files changed, 86 insertions(+), 10 deletions(-) create mode 100644 packages/pandas-gbq/tests/system/test_to_gbq.py diff --git a/packages/pandas-gbq/conftest.py b/packages/pandas-gbq/conftest.py index 7f9a67215f55..b5803f374303 100644 --- a/packages/pandas-gbq/conftest.py +++ b/packages/pandas-gbq/conftest.py @@ -1,4 +1,4 @@ -"""Shared pytest fixtures for system tests.""" +"""Shared pytest fixtures for `tests/system` and `samples/tests` tests.""" import os import os.path diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 46570643c886..d4c520442c52 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,6 +1,17 @@ Changelog ========= +.. _changelog-0.14.1: + +0.14.1 / TBD +------------ + +Bug fixes +~~~~~~~~~ + +- Encode floating point values with greater precision. (:issue:`326`) + + .. _changelog-0.14.0: 0.14.0 / 2020-10-05 diff --git a/packages/pandas-gbq/pandas_gbq/load.py b/packages/pandas-gbq/pandas_gbq/load.py index 04b32efaf983..ec00d4a12a93 100644 --- a/packages/pandas-gbq/pandas_gbq/load.py +++ b/packages/pandas-gbq/pandas_gbq/load.py @@ -19,7 +19,7 @@ def encode_chunk(dataframe): index=False, header=False, encoding="utf-8", - float_format="%.15g", + float_format="%.17g", date_format="%Y-%m-%d %H:%M:%S.%f", ) diff --git a/packages/pandas-gbq/tests/system/test_to_gbq.py b/packages/pandas-gbq/tests/system/test_to_gbq.py new file mode 100644 index 000000000000..ca5e406aa43c --- /dev/null +++ b/packages/pandas-gbq/tests/system/test_to_gbq.py @@ -0,0 +1,49 @@ +import functools +import pandas +import pandas.testing + +import pytest + + +pytest.importorskip("google.cloud.bigquery", minversion="1.24.0") + + +@pytest.fixture +def method_under_test(credentials): + import pandas_gbq + + return functools.partial(pandas_gbq.to_gbq, credentials=credentials) + + +def test_float_round_trip( + method_under_test, random_dataset_id, bigquery_client +): + """Ensure that 64-bit floating point numbers are unchanged. + + See: https://github.com/pydata/pandas-gbq/issues/326 + """ + + table_id = "{}.float_round_trip".format(random_dataset_id) + input_floats = pandas.Series( + [ + 0.14285714285714285, + 0.4406779661016949, + 1.05148, + 1.05153, + 1.8571428571428572, + 2.718281828459045, + 3.141592653589793, + 2.0988936657440586e43, + ], + name="float_col", + ) + df = pandas.DataFrame({"float_col": input_floats}) + method_under_test(df, table_id) + + round_trip = bigquery_client.list_rows(table_id).to_dataframe() + round_trip_floats = round_trip["float_col"].sort_values() + pandas.testing.assert_series_equal( + round_trip_floats, + input_floats, + check_exact=True, + ) diff --git a/packages/pandas-gbq/tests/unit/test_load.py b/packages/pandas-gbq/tests/unit/test_load.py index 9be8fe89fcaf..7ed463c18850 100644 --- a/packages/pandas-gbq/tests/unit/test_load.py +++ b/packages/pandas-gbq/tests/unit/test_load.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +import textwrap from io import StringIO import numpy @@ -24,17 +25,32 @@ def test_encode_chunk_with_unicode(): def test_encode_chunk_with_floats(): - """Test that floats in a dataframe are encoded with at most 15 significant + """Test that floats in a dataframe are encoded with at most 17 significant figures. - See: https://github.com/pydata/pandas-gbq/issues/192 + See: https://github.com/pydata/pandas-gbq/issues/192 and + https://github.com/pydata/pandas-gbq/issues/326 """ - input_csv = StringIO(u"01/01/17 23:00,1.05148,1.05153,1.05148,1.05153,4") - df = pandas.read_csv(input_csv, header=None) - csv_buffer = load.encode_chunk(df) - csv_bytes = csv_buffer.read() - csv_string = csv_bytes.decode("utf-8") - assert "1.05153" in csv_string + input_csv = textwrap.dedent( + """01/01/17 23:00,0.14285714285714285,4 + 01/02/17 22:00,1.05148,3 + 01/03/17 21:00,1.05153,2 + 01/04/17 20:00,3.141592653589793,1 + 01/05/17 19:00,2.0988936657440586e+43,0 + """ + ) + input_df = pandas.read_csv( + StringIO(input_csv), header=None, float_precision="round_trip" + ) + csv_buffer = load.encode_chunk(input_df) + round_trip = pandas.read_csv( + csv_buffer, header=None, float_precision="round_trip" + ) + pandas.testing.assert_frame_equal( + round_trip, + input_df, + check_exact=True, + ) def test_encode_chunk_with_newlines(): From 3a3e07ceba04723e901d4bfcc3841dc4d33fc8bc Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 9 Nov 2020 16:05:28 -0600 Subject: [PATCH 213/519] BUG: support INT64 and other standard SQL aliases in to_gbq table_schema (#340) --- packages/pandas-gbq/docs/source/changelog.rst | 2 + packages/pandas-gbq/pandas_gbq/schema.py | 20 ++++++-- packages/pandas-gbq/tests/unit/test_schema.py | 50 +++++++++++++------ 3 files changed, 54 insertions(+), 18 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index d4c520442c52..486b547b9b65 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -10,6 +10,8 @@ Bug fixes ~~~~~~~~~ - Encode floating point values with greater precision. (:issue:`326`) +- Support ``INT64`` and other standard SQL aliases in + :func:`~pandas_gbq.to_gbq` ``table_schema`` argument. (:issue:`322`) .. _changelog-0.14.0: diff --git a/packages/pandas-gbq/pandas_gbq/schema.py b/packages/pandas-gbq/pandas_gbq/schema.py index 4154044bf74c..ffc1c36280cd 100644 --- a/packages/pandas-gbq/pandas_gbq/schema.py +++ b/packages/pandas-gbq/pandas_gbq/schema.py @@ -3,6 +3,16 @@ import copy +# API may return data types as legacy SQL, so maintain a mapping of aliases +# from standard SQL to legacy data types. +_TYPE_ALIASES = { + "BOOL": "BOOLEAN", + "FLOAT64": "FLOAT", + "INT64": "INTEGER", + "STRUCT": "RECORD", +} + + def to_pandas_gbq(client_schema): """Given a sequence of :class:`google.cloud.bigquery.schema.SchemaField`, return a schema in pandas-gbq API format. @@ -24,10 +34,12 @@ def _clean_schema_fields(fields): are not generated by func:`pandas_gbq.schema.generate_bq_schema`. """ fields_sorted = sorted(fields, key=lambda field: field["name"]) - return [ - {"name": field["name"], "type": field["type"]} - for field in fields_sorted - ] + clean_schema = [] + for field in fields_sorted: + field_type = field["type"].upper() + field_type = _TYPE_ALIASES.get(field_type, field_type) + clean_schema.append({"name": field["name"], "type": field_type}) + return clean_schema def schema_is_subset(schema_remote, schema_local): diff --git a/packages/pandas-gbq/tests/unit/test_schema.py b/packages/pandas-gbq/tests/unit/test_schema.py index 2b7324281fef..16286a2d412d 100644 --- a/packages/pandas-gbq/tests/unit/test_schema.py +++ b/packages/pandas-gbq/tests/unit/test_schema.py @@ -11,22 +11,44 @@ def module_under_test(): return pandas_gbq.schema -def test_schema_is_subset_passes_if_subset(module_under_test): +@pytest.mark.parametrize( + "original_fields,dataframe_fields", + [ + ( + [ + {"name": "A", "type": "FLOAT"}, + {"name": "B", "type": "FLOAT64"}, + {"name": "C", "type": "STRING"}, + ], + [ + {"name": "A", "type": "FLOAT64"}, + {"name": "B", "type": "FLOAT"}, + ], + ), + # Original schema from API may contain legacy SQL datatype names. + # https://github.com/pydata/pandas-gbq/issues/322 + ( + [{"name": "A", "type": "INTEGER"}], + [{"name": "A", "type": "INT64"}], + ), + ( + [{"name": "A", "type": "BOOL"}], + [{"name": "A", "type": "BOOLEAN"}], + ), + ( + # TODO: include sub-fields when struct uploads are supported. + [{"name": "A", "type": "STRUCT"}], + [{"name": "A", "type": "RECORD"}], + ), + ], +) +def test_schema_is_subset_passes_if_subset( + module_under_test, original_fields, dataframe_fields +): # Issue #24 schema_is_subset indicates whether the schema of the # dataframe is a subset of the schema of the bigquery table - table_schema = { - "fields": [ - {"name": "A", "type": "FLOAT"}, - {"name": "B", "type": "FLOAT"}, - {"name": "C", "type": "STRING"}, - ] - } - tested_schema = { - "fields": [ - {"name": "A", "type": "FLOAT"}, - {"name": "B", "type": "FLOAT"}, - ] - } + table_schema = {"fields": original_fields} + tested_schema = {"fields": dataframe_fields} assert module_under_test.schema_is_subset(table_schema, tested_schema) From 001a35af29c4fe210e42b2a515e7752ba8ebf03c Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 9 Nov 2020 17:19:00 -0600 Subject: [PATCH 214/519] BUG: use object dtype for TIME columns (#341) --- packages/pandas-gbq/docs/source/changelog.rst | 1 + packages/pandas-gbq/pandas_gbq/gbq.py | 4 +- packages/pandas-gbq/tests/system/test_gbq.py | 39 +++++++++++++++++-- 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 486b547b9b65..bb9e7eaac9a4 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -9,6 +9,7 @@ Changelog Bug fixes ~~~~~~~~~ +- Use ``object`` dtype for ``TIME`` columns. (:issue:`328`) - Encode floating point values with greater precision. (:issue:`326`) - Support ``INT64`` and other standard SQL aliases in :func:`~pandas_gbq.to_gbq` ``table_schema`` argument. (:issue:`322`) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 1b71a7a38cfe..11b5c7b9e402 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -725,7 +725,9 @@ def _bqschema_to_nullsafe_dtypes(schema_fields): "GEOMETRY": "object", "RECORD": "object", "STRING": "object", - "TIME": "datetime64[ns]", + # datetime.time objects cannot be case to datetime64. + # https://github.com/pydata/pandas-gbq/issues/328 + "TIME": "object", # pandas doesn't support timezone-aware dtype in DataFrame/Series # constructors. It's more idiomatic to localize after construction. # https://github.com/pandas-dev/pandas/issues/25843 diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index c583b48baddc..a11d2e6ee4cf 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- +import datetime import sys -from datetime import datetime import numpy as np import pandas @@ -39,7 +39,8 @@ def make_mixed_dataframe_v2(test_size): ints = np.random.randint(1, 10, size=(1, test_size)) strs = np.random.randint(1, 10, size=(1, test_size)).astype(str) times = [ - datetime.now(pytz.timezone("US/Arizona")) for t in range(test_size) + datetime.datetime.now(pytz.timezone("US/Arizona")) + for t in range(test_size) ] return DataFrame( { @@ -248,6 +249,38 @@ def test_should_properly_handle_null_floats(self, project_id): ) tm.assert_frame_equal(df, DataFrame({"null_float": [np.nan, 1.0]})) + def test_should_properly_handle_date(self, project_id): + query = "SELECT DATE(2003, 1, 4) AS date_col" + df = gbq.read_gbq( + query, + project_id=project_id, + credentials=self.credentials, + ) + expected = DataFrame( + { + "date_col": pandas.Series( + [datetime.date(2003, 1, 4)], dtype="datetime64[ns]" + ) + }, + ) + tm.assert_frame_equal(df, expected) + + def test_should_properly_handle_time(self, project_id): + query = "SELECT TIME_ADD(TIME(3, 14, 15), INTERVAL 926589 MICROSECOND) AS time_col" + df = gbq.read_gbq( + query, + project_id=project_id, + credentials=self.credentials, + ) + expected = DataFrame( + { + "time_col": pandas.Series( + [datetime.time(3, 14, 15, 926589)], dtype="object" + ) + }, + ) + tm.assert_frame_equal(df, expected) + def test_should_properly_handle_timestamp_unix_epoch(self, project_id): query = 'SELECT TIMESTAMP("1970-01-01 00:00:00") AS unix_epoch' df = gbq.read_gbq( @@ -1113,7 +1146,7 @@ def test_google_upload_errors_should_raise_exception(self, project_id): raise pytest.skip("buggy test") test_id = "5" - test_timestamp = datetime.now(pytz.timezone("US/Arizona")) + test_timestamp = datetime.datetime.now(pytz.timezone("US/Arizona")) bad_df = DataFrame( { "bools": [False, False], From eb375c338f626dc3e8aecef87f45c741ac978dcd Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 10 Nov 2020 11:02:50 -0600 Subject: [PATCH 215/519] release 0.14.1 --- packages/pandas-gbq/docs/source/changelog.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index bb9e7eaac9a4..20ef4139327f 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -3,8 +3,8 @@ Changelog .. _changelog-0.14.1: -0.14.1 / TBD ------------- +0.14.1 / 2020-11-10 +------------------- Bug fixes ~~~~~~~~~ From b0762479665d87d8ad04fe1ac40be699ee4031c3 Mon Sep 17 00:00:00 2001 From: Vicente Reyes-Puerta Date: Fri, 4 Dec 2020 17:32:54 +0100 Subject: [PATCH 216/519] ENH: add project id to destination table in to_gbq() (#347) * ENH: add project id to destination table in to_gbq() * ENH: fix non-callable client error when adding project id to destination table * Update pandas_gbq/gbq.py (pass table reference) Co-authored-by: Tim Swast * Update pandas_gbq/load.py (pass destination table) Co-authored-by: Tim Swast * Update pandas_gbq/load.py (delete unnecessary variable) Co-authored-by: Tim Swast * Update pandas_gbq/gbq.py (pass destination_table_ref) Co-authored-by: Tim Swast * Fix call to load.load_chunks (now using only destination_table_ref) * add assertions for project ID to unit test * add to changelog * use project from credentials if none provided Co-authored-by: Tim Swast --- packages/pandas-gbq/docs/source/changelog.rst | 13 ++++++ packages/pandas-gbq/pandas_gbq/gbq.py | 42 ++++++++++++------ packages/pandas-gbq/pandas_gbq/load.py | 6 +-- packages/pandas-gbq/tests/unit/test_gbq.py | 44 +++++++++++++++++++ 4 files changed, 87 insertions(+), 18 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 20ef4139327f..0dd1f925660f 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -1,6 +1,19 @@ Changelog ========= +.. _changelog-0.15.0: + +0.15.0 / TBD +------------ + +Features +~~~~~~~~ + +- Load DataFrame with ``to_gbq`` to a table in a project different from the API + client project. Specify the target table ID as ``project.dataset.table`` to + use this feature. (:issue:`321`, :issue:`347`) + + .. _changelog-0.14.1: 0.14.1 / 2020-11-10 diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 11b5c7b9e402..bebe7e1e3e81 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -604,8 +604,7 @@ def _download_results( def load_data( self, dataframe, - dataset_id, - table_id, + destination_table_ref, chunksize=None, schema=None, progress_bar=True, @@ -618,8 +617,7 @@ def load_data( chunks = load.load_chunks( self.client, dataframe, - dataset_id, - table_id, + destination_table_ref, chunksize=chunksize, schema=schema, location=self.location, @@ -1037,7 +1035,8 @@ def to_gbq( dataframe : pandas.DataFrame DataFrame to be written to a Google BigQuery table. destination_table : str - Name of table to be written, in the form ``dataset.tablename``. + Name of table to be written, in the form ``dataset.tablename`` or + ``project.dataset.tablename``. project_id : str, optional Google BigQuery Account project ID. Optional when available from the environment. @@ -1133,7 +1132,8 @@ def to_gbq( if "." not in destination_table: raise NotFoundException( - "Invalid Table Name. Should be of the form 'datasetId.tableId' " + "Invalid Table Name. Should be of the form 'datasetId.tableId' or " + "'projectId.datasetId.tableId'" ) connector = GbqConnector( @@ -1145,7 +1145,14 @@ def to_gbq( private_key=private_key, ) bqclient = connector.client - dataset_id, table_id = destination_table.rsplit(".", 1) + + destination_table_ref = bigquery.table.TableReference.from_string( + destination_table, default_project=connector.project_id + ) + + project_id_table = destination_table_ref.project + dataset_id = destination_table_ref.dataset_id + table_id = destination_table_ref.table_id default_schema = _generate_bq_schema(dataframe) if not table_schema: @@ -1157,10 +1164,10 @@ def to_gbq( # If table exists, check if_exists parameter try: - table = bqclient.get_table(destination_table) + table = bqclient.get_table(destination_table_ref) except google_exceptions.NotFound: table_connector = _Table( - project_id, + project_id_table, dataset_id, location=location, credentials=connector.credentials, @@ -1203,8 +1210,7 @@ def to_gbq( connector.load_data( dataframe, - dataset_id, - table_id, + destination_table_ref, chunksize=chunksize, schema=table_schema, progress_bar=progress_bar, @@ -1279,8 +1285,12 @@ def exists(self, table_id): true if table exists, otherwise false """ from google.api_core.exceptions import NotFound + from google.cloud.bigquery import DatasetReference + from google.cloud.bigquery import TableReference - table_ref = self.client.dataset(self.dataset_id).table(table_id) + table_ref = TableReference( + DatasetReference(self.project_id, self.dataset_id), table_id + ) try: self.client.get_table(table_ref) return True @@ -1300,12 +1310,14 @@ def create(self, table_id, schema): Use the generate_bq_schema to generate your table schema from a dataframe. """ + from google.cloud.bigquery import DatasetReference from google.cloud.bigquery import SchemaField from google.cloud.bigquery import Table + from google.cloud.bigquery import TableReference if self.exists(table_id): raise TableCreationError( - "Table {0} already " "exists".format(table_id) + "Table {0} already exists".format(table_id) ) if not _Dataset(self.project_id, credentials=self.credentials).exists( @@ -1317,7 +1329,9 @@ def create(self, table_id, schema): location=self.location, ).create(self.dataset_id) - table_ref = self.client.dataset(self.dataset_id).table(table_id) + table_ref = TableReference( + DatasetReference(self.project_id, self.dataset_id), table_id + ) table = Table(table_ref) schema = pandas_gbq.schema.add_default_nullable_mode(schema) diff --git a/packages/pandas-gbq/pandas_gbq/load.py b/packages/pandas-gbq/pandas_gbq/load.py index ec00d4a12a93..d9e59c1d16eb 100644 --- a/packages/pandas-gbq/pandas_gbq/load.py +++ b/packages/pandas-gbq/pandas_gbq/load.py @@ -50,13 +50,11 @@ def encode_chunks(dataframe, chunksize=None): def load_chunks( client, dataframe, - dataset_id, - table_id, + destination_table_ref, chunksize=None, schema=None, location=None, ): - destination_table = client.dataset(dataset_id).table(table_id) job_config = bigquery.LoadJobConfig() job_config.write_disposition = "WRITE_APPEND" job_config.source_format = "CSV" @@ -77,7 +75,7 @@ def load_chunks( yield remaining_rows client.load_table_from_file( chunk_buffer, - destination_table, + destination_table_ref, job_config=job_config, location=location, ).result() diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index d2a41608b2ce..4426e8dc84c9 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -257,6 +257,50 @@ def test_to_gbq_w_empty_df(mock_bigquery_client): mock_bigquery_client.load_table_from_file.assert_not_called() +def test_to_gbq_w_default_project(mock_bigquery_client): + """If no project is specified, we should be able to use project from + default credentials. + """ + import google.api_core.exceptions + from google.cloud.bigquery.table import TableReference + + mock_bigquery_client.get_table.side_effect = ( + google.api_core.exceptions.NotFound("my_table") + ) + gbq.to_gbq(DataFrame(), "my_dataset.my_table") + + mock_bigquery_client.get_table.assert_called_with( + TableReference.from_string("default-project.my_dataset.my_table") + ) + mock_bigquery_client.create_table.assert_called_with(mock.ANY) + table = mock_bigquery_client.create_table.call_args[0][0] + assert table.project == "default-project" + + +def test_to_gbq_w_project_table(mock_bigquery_client): + """If a project is included in the table ID, use that instead of the client + project. See: https://github.com/pydata/pandas-gbq/issues/321 + """ + import google.api_core.exceptions + from google.cloud.bigquery.table import TableReference + + mock_bigquery_client.get_table.side_effect = ( + google.api_core.exceptions.NotFound("my_table") + ) + gbq.to_gbq( + DataFrame(), + "project_table.my_dataset.my_table", + project_id="project_client", + ) + + mock_bigquery_client.get_table.assert_called_with( + TableReference.from_string("project_table.my_dataset.my_table") + ) + mock_bigquery_client.create_table.assert_called_with(mock.ANY) + table = mock_bigquery_client.create_table.call_args[0][0] + assert table.project == "project_table" + + def test_to_gbq_creates_dataset(mock_bigquery_client): import google.api_core.exceptions From 0250e6d6ccc8b5c48ff2af6772e33dc94f6778e9 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 17 Dec 2020 15:26:00 -0600 Subject: [PATCH 217/519] deps: drop Python 3.5 and 3.6 (#342) * deps: support Python 3.9, drop 3.5 and 3.6 * update minimum pandas * drop 3.9 * drop 3.9 * add docs-presubmit to build * fix lint session --- packages/pandas-gbq/.circleci/config.yml | 52 ++++++++----------- packages/pandas-gbq/ci/constraints-3.5.pip | 6 --- packages/pandas-gbq/ci/constraints-3.6.pip | 3 -- packages/pandas-gbq/ci/constraints-3.7.pip | 11 ++-- packages/pandas-gbq/ci/constraints-3.8.pip | 2 +- packages/pandas-gbq/ci/constraints-3.9.pip | 2 + ....1.conda => requirements-3.7-0.23.2.conda} | 0 ...Y.conda => requirements-3.9-NIGHTLY.conda} | 0 packages/pandas-gbq/docs/source/changelog.rst | 5 ++ packages/pandas-gbq/noxfile.py | 13 +++-- packages/pandas-gbq/setup.py | 7 ++- 11 files changed, 47 insertions(+), 54 deletions(-) delete mode 100644 packages/pandas-gbq/ci/constraints-3.5.pip delete mode 100644 packages/pandas-gbq/ci/constraints-3.6.pip create mode 100644 packages/pandas-gbq/ci/constraints-3.9.pip rename packages/pandas-gbq/ci/{requirements-3.6-0.20.1.conda => requirements-3.7-0.23.2.conda} (100%) rename packages/pandas-gbq/ci/{requirements-3.8-NIGHTLY.conda => requirements-3.9-NIGHTLY.conda} (100%) diff --git a/packages/pandas-gbq/.circleci/config.yml b/packages/pandas-gbq/.circleci/config.yml index a3863bd44cc2..817b2e73f083 100644 --- a/packages/pandas-gbq/.circleci/config.yml +++ b/packages/pandas-gbq/.circleci/config.yml @@ -1,22 +1,27 @@ version: 2 jobs: - # Pip - "pip-3.5": + "lint": docker: - image: thekevjames/nox + environment: + # Resolve "Python 3 was configured to use ASCII as encoding for the environment" + LC_ALL: C.UTF-8 + LANG: C.UTF-8 steps: - checkout - - run: ci/config_auth.sh - - run: nox -s unit-3.5 system-3.5 - - "pip-3.6": + - run: nox -s lint + "docs-presubmit": docker: - image: thekevjames/nox + environment: + # Resolve "Python 3 was configured to use ASCII as encoding for the environment" + LC_ALL: C.UTF-8 + LANG: C.UTF-8 steps: - checkout - - run: ci/config_auth.sh - - run: nox -s unit-3.6 + - run: nox -s docs + # Pip "pip-3.7": docker: - image: thekevjames/nox @@ -24,7 +29,6 @@ jobs: - checkout - run: ci/config_auth.sh - run: nox -s unit-3.7 - "pip-3.8": docker: - image: thekevjames/nox @@ -33,33 +37,22 @@ jobs: - run: ci/config_auth.sh - run: nox -s unit-3.8 system-3.8 cover - "lint": - docker: - - image: thekevjames/nox - environment: - # Resolve "Python 3 was configured to use ASCII as encoding for the environment" - LC_ALL: C.UTF-8 - LANG: C.UTF-8 - steps: - - checkout - - run: nox -s lint - # Conda - "conda-3.6-0.20.1": + "conda-3.7": docker: - image: continuumio/miniconda3 environment: - PYTHON: "3.6" - PANDAS: "0.20.1" + PYTHON: "3.7" + PANDAS: "0.23.2" steps: - checkout - run: ci/config_auth.sh - run: ci/run_conda.sh - "conda-3.8-NIGHTLY": + "conda-3.9-NIGHTLY": docker: - image: continuumio/miniconda3 environment: - PYTHON: "3.8" + PYTHON: "3.9" PANDAS: "NIGHTLY" steps: - checkout @@ -70,10 +63,9 @@ workflows: version: 2 build: jobs: - - "pip-3.5" - - "pip-3.6" + - lint + - docs-presubmit - "pip-3.7" - "pip-3.8" - - lint - - "conda-3.6-0.20.1" - - "conda-3.8-NIGHTLY" \ No newline at end of file + - "conda-3.7" + - "conda-3.9-NIGHTLY" \ No newline at end of file diff --git a/packages/pandas-gbq/ci/constraints-3.5.pip b/packages/pandas-gbq/ci/constraints-3.5.pip deleted file mode 100644 index 2fd6f28a708d..000000000000 --- a/packages/pandas-gbq/ci/constraints-3.5.pip +++ /dev/null @@ -1,6 +0,0 @@ -numpy==1.13.3 -pandas==0.20.1 -google-auth==1.4.1 -google-auth-oauthlib==0.0.1 -google-cloud-bigquery==1.11.1 -pydata-google-auth==0.1.2 \ No newline at end of file diff --git a/packages/pandas-gbq/ci/constraints-3.6.pip b/packages/pandas-gbq/ci/constraints-3.6.pip deleted file mode 100644 index 25b7b34ec28f..000000000000 --- a/packages/pandas-gbq/ci/constraints-3.6.pip +++ /dev/null @@ -1,3 +0,0 @@ -pandas -pydata-google-auth -google-cloud-bigquery \ No newline at end of file diff --git a/packages/pandas-gbq/ci/constraints-3.7.pip b/packages/pandas-gbq/ci/constraints-3.7.pip index 6025ac8a4c4d..3477a2b7ad07 100644 --- a/packages/pandas-gbq/ci/constraints-3.7.pip +++ b/packages/pandas-gbq/ci/constraints-3.7.pip @@ -1,3 +1,8 @@ -pandas==0.24.0 -google-cloud-bigquery==1.12.0 -pydata-google-auth==0.1.2 \ No newline at end of file +numpy==1.14.5 +pandas==0.23.2 +google-auth==1.4.1 +google-auth-oauthlib==0.0.1 +google-cloud-bigquery==1.11.1 +google-cloud-bigquery[bqstorage,pandas]==1.11.1 +pydata-google-auth==0.1.2 +tqdm==4.23.0 diff --git a/packages/pandas-gbq/ci/constraints-3.8.pip b/packages/pandas-gbq/ci/constraints-3.8.pip index 1411a4a0b5ab..9c67e95ef27d 100644 --- a/packages/pandas-gbq/ci/constraints-3.8.pip +++ b/packages/pandas-gbq/ci/constraints-3.8.pip @@ -1 +1 @@ -pandas \ No newline at end of file +numpy==1.17.5 diff --git a/packages/pandas-gbq/ci/constraints-3.9.pip b/packages/pandas-gbq/ci/constraints-3.9.pip new file mode 100644 index 000000000000..76864a661daf --- /dev/null +++ b/packages/pandas-gbq/ci/constraints-3.9.pip @@ -0,0 +1,2 @@ +numpy==1.19.4 +pandas==1.1.4 diff --git a/packages/pandas-gbq/ci/requirements-3.6-0.20.1.conda b/packages/pandas-gbq/ci/requirements-3.7-0.23.2.conda similarity index 100% rename from packages/pandas-gbq/ci/requirements-3.6-0.20.1.conda rename to packages/pandas-gbq/ci/requirements-3.7-0.23.2.conda diff --git a/packages/pandas-gbq/ci/requirements-3.8-NIGHTLY.conda b/packages/pandas-gbq/ci/requirements-3.9-NIGHTLY.conda similarity index 100% rename from packages/pandas-gbq/ci/requirements-3.8-NIGHTLY.conda rename to packages/pandas-gbq/ci/requirements-3.9-NIGHTLY.conda diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 0dd1f925660f..372a0c61760e 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -13,6 +13,11 @@ Features client project. Specify the target table ID as ``project.dataset.table`` to use this feature. (:issue:`321`, :issue:`347`) +Dependencies +~~~~~~~~~~~~ + +- Drop support for Python 3.5 and 3.6. (:issue:`337`) + .. _changelog-0.14.1: diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 6ed0cef1cbe9..0af7c2f6b9b4 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -10,8 +10,8 @@ import nox -supported_pythons = ["3.5", "3.6", "3.7", "3.8"] -system_test_pythons = ["3.5", "3.8"] +supported_pythons = ["3.7", "3.8"] +system_test_pythons = ["3.7", "3.8"] latest_python = "3.8" # Use a consistent version of black so CI is deterministic. @@ -19,10 +19,9 @@ black_package = "black==20.8b1" -@nox.session -def lint(session, python=latest_python): +@nox.session(python=latest_python) +def lint(session): session.install(black_package, "flake8") - session.install("-e", ".") session.run("flake8", "pandas_gbq") session.run("flake8", "tests") session.run("black", "--check", ".") @@ -57,8 +56,8 @@ def unit(session): ) -@nox.session -def cover(session, python=latest_python): +@nox.session(python=latest_python) +def cover(session): session.install("coverage", "pytest-cov") session.run("coverage", "report", "--show-missing", "--fail-under=73") session.run("coverage", "erase") diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index 4e3d01c9b5dc..3ebe4a46cc73 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -18,7 +18,7 @@ def readme(): INSTALL_REQUIRES = [ "setuptools", - "pandas>=0.20.1", + "pandas>=0.23.2", "pydata-google-auth", "google-auth", "google-auth-oauthlib", @@ -44,16 +44,15 @@ def readme(): "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", "Topic :: Scientific/Engineering", ], keywords="data", install_requires=INSTALL_REQUIRES, extras_require=extras, - python_requires=">=3.5", + python_requires=">=3.7", packages=find_packages(exclude=["contrib", "docs", "tests*"]), test_suite="tests", ) From 97ae7c7ae7b274843dbee1622737c8171633e6ea Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 19 Jan 2021 10:28:47 -0600 Subject: [PATCH 218/519] FIX: remove references to deprecated client.dataset method (#355) * FIX: remove references to deprecated client.dataset method * fix lint * fix constraints file --- packages/pandas-gbq/ci/constraints-3.7.pip | 2 +- packages/pandas-gbq/pandas_gbq/gbq.py | 91 +++--------- packages/pandas-gbq/tests/system/test_gbq.py | 145 +++++++++++++++---- 3 files changed, 136 insertions(+), 102 deletions(-) diff --git a/packages/pandas-gbq/ci/constraints-3.7.pip b/packages/pandas-gbq/ci/constraints-3.7.pip index 3477a2b7ad07..362dfc850a5f 100644 --- a/packages/pandas-gbq/ci/constraints-3.7.pip +++ b/packages/pandas-gbq/ci/constraints-3.7.pip @@ -3,6 +3,6 @@ pandas==0.23.2 google-auth==1.4.1 google-auth-oauthlib==0.0.1 google-cloud-bigquery==1.11.1 -google-cloud-bigquery[bqstorage,pandas]==1.11.1 +google-cloud-bigquery-storage==1.1.0 pydata-google-auth==0.1.2 tqdm==4.23.0 diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index bebe7e1e3e81..8d7c15c84b7c 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -633,70 +633,6 @@ def load_data( except self.http_error as ex: self.process_http_error(ex) - def schema(self, dataset_id, table_id): - """Retrieve the schema of the table - - Obtain from BigQuery the field names and field types - for the table defined by the parameters - - Parameters - ---------- - dataset_id : str - Name of the BigQuery dataset for the table - table_id : str - Name of the BigQuery table - - Returns - ------- - list of dicts - Fields representing the schema - """ - table_ref = self.client.dataset(dataset_id).table(table_id) - - try: - table = self.client.get_table(table_ref) - remote_schema = table.schema - - remote_fields = [ - field_remote.to_api_repr() for field_remote in remote_schema - ] - for field in remote_fields: - field["type"] = field["type"].upper() - field["mode"] = field["mode"].upper() - - return remote_fields - except self.http_error as ex: - self.process_http_error(ex) - - def verify_schema(self, dataset_id, table_id, schema): - """Indicate whether schemas match exactly - - Compare the BigQuery table identified in the parameters with - the schema passed in and indicate whether all fields in the former - are present in the latter. Order is not considered. - - Parameters - ---------- - dataset_id :str - Name of the BigQuery dataset for the table - table_id : str - Name of the BigQuery table - schema : list(dict) - Schema for comparison. Each item should have - a 'name' and a 'type' - - Returns - ------- - bool - Whether the schemas match - """ - - fields_remote = pandas_gbq.schema._clean_schema_fields( - self.schema(dataset_id, table_id) - ) - fields_local = pandas_gbq.schema._clean_schema_fields(schema["fields"]) - return fields_remote == fields_local - def delete_and_recreate_table(self, dataset_id, table_id, table_schema): table = _Table( self.project_id, dataset_id, credentials=self.credentials @@ -1271,6 +1207,15 @@ def __init__( private_key=private_key, ) + def _table_ref(self, table_id): + """Return a BigQuery client library table reference""" + from google.cloud.bigquery import DatasetReference + from google.cloud.bigquery import TableReference + + return TableReference( + DatasetReference(self.project_id, self.dataset_id), table_id + ) + def exists(self, table_id): """Check if a table exists in Google BigQuery @@ -1285,12 +1230,8 @@ def exists(self, table_id): true if table exists, otherwise false """ from google.api_core.exceptions import NotFound - from google.cloud.bigquery import DatasetReference - from google.cloud.bigquery import TableReference - table_ref = TableReference( - DatasetReference(self.project_id, self.dataset_id), table_id - ) + table_ref = self._table_ref(table_id) try: self.client.get_table(table_ref) return True @@ -1358,7 +1299,7 @@ def delete(self, table_id): if not self.exists(table_id): raise NotFoundException("Table does not exist") - table_ref = self.client.dataset(self.dataset_id).table(table_id) + table_ref = self._table_ref(table_id) try: self.client.delete_table(table_ref) except NotFound: @@ -1385,6 +1326,12 @@ def __init__( private_key=private_key, ) + def _dataset_ref(self, dataset_id): + """Return a BigQuery client library dataset reference""" + from google.cloud.bigquery import DatasetReference + + return DatasetReference(self.project_id, dataset_id) + def exists(self, dataset_id): """Check if a dataset exists in Google BigQuery @@ -1401,7 +1348,7 @@ def exists(self, dataset_id): from google.api_core.exceptions import NotFound try: - self.client.get_dataset(self.client.dataset(dataset_id)) + self.client.get_dataset(self._dataset_ref(dataset_id)) return True except NotFound: return False @@ -1423,7 +1370,7 @@ def create(self, dataset_id): "Dataset {0} already " "exists".format(dataset_id) ) - dataset = Dataset(self.client.dataset(dataset_id)) + dataset = Dataset(self._dataset_ref(dataset_id)) if self.location is not None: dataset.location = self.location diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index a11d2e6ee4cf..037ff3a6b9f1 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -54,6 +54,80 @@ def make_mixed_dataframe_v2(test_size): ) +def get_schema( + gbq_connector: gbq.GbqConnector, dataset_id: str, table_id: str +): + """Retrieve the schema of the table + + Obtain from BigQuery the field names and field types + for the table defined by the parameters + + Parameters + ---------- + dataset_id : str + Name of the BigQuery dataset for the table + table_id : str + Name of the BigQuery table + + Returns + ------- + list of dicts + Fields representing the schema + """ + from google.cloud import bigquery + + bqclient = gbq_connector.client + table_ref = bigquery.TableReference( + bigquery.DatasetReference(bqclient.project, dataset_id), + table_id, + ) + + try: + table = bqclient.get_table(table_ref) + remote_schema = table.schema + + remote_fields = [ + field_remote.to_api_repr() for field_remote in remote_schema + ] + for field in remote_fields: + field["type"] = field["type"].upper() + field["mode"] = field["mode"].upper() + + return remote_fields + except gbq_connector.http_error as ex: + gbq_connector.process_http_error(ex) + + +def verify_schema(gbq_connector, dataset_id, table_id, schema): + """Indicate whether schemas match exactly + + Compare the BigQuery table identified in the parameters with + the schema passed in and indicate whether all fields in the former + are present in the latter. Order is not considered. + + Parameters + ---------- + dataset_id :str + Name of the BigQuery dataset for the table + table_id : str + Name of the BigQuery table + schema : list(dict) + Schema for comparison. Each item should have + a 'name' and a 'type' + + Returns + ------- + bool + Whether the schemas match + """ + + fields_remote = pandas_gbq.schema._clean_schema_fields( + get_schema(gbq_connector, dataset_id, table_id) + ) + fields_local = pandas_gbq.schema._clean_schema_fields(schema["fields"]) + return fields_remote == fields_local + + class TestGBQConnectorIntegration(object): def test_should_be_able_to_make_a_connector(self, gbq_connector): assert gbq_connector is not None, "Could not create a GbqConnector" @@ -914,19 +988,16 @@ def test_tokyo(self, tokyo_dataset, tokyo_table, project_id): class TestToGBQIntegration(object): @pytest.fixture(autouse=True, scope="function") def setup(self, project, credentials, random_dataset_id): - from google.cloud import bigquery - # - PER-TEST FIXTURES - # put here any instruction you want to be run *BEFORE* *EVERY* test is # executed. + self.credentials = credentials + self.gbq_connector = gbq.GbqConnector(project, credentials=credentials) + self.bqclient = self.gbq_connector.client self.table = gbq._Table( project, random_dataset_id, credentials=credentials ) self.destination_table = "{}.{}".format(random_dataset_id, TABLE_ID) - self.credentials = credentials - self.bqclient = bigquery.Client( - project=project, credentials=credentials - ) def test_upload_data(self, project_id): test_id = "1" @@ -1340,8 +1411,8 @@ def test_upload_data_with_valid_user_schema(self, project_id): table_schema=test_schema, ) dataset, table = destination_table.split(".") - assert self.table.verify_schema( - dataset, table, dict(fields=test_schema) + assert verify_schema( + self.gbq_connector, dataset, table, dict(fields=test_schema) ) def test_upload_data_with_invalid_user_schema_raises_error( @@ -1445,13 +1516,15 @@ def test_upload_data_with_different_df_and_user_schema(self, project_id): table_schema=test_schema, ) dataset, table = destination_table.split(".") - assert self.table.verify_schema( - dataset, table, dict(fields=test_schema) + assert verify_schema( + self.gbq_connector, dataset, table, dict(fields=test_schema) ) def test_upload_data_tokyo( self, project_id, tokyo_dataset, bigquery_client ): + from google.cloud import bigquery + test_size = 10 df = make_mixed_dataframe_v2(test_size) tokyo_destination = "{}.to_gbq_test".format(tokyo_dataset) @@ -1466,13 +1539,18 @@ def test_upload_data_tokyo( ) table = bigquery_client.get_table( - bigquery_client.dataset(tokyo_dataset).table("to_gbq_test") + bigquery.TableReference( + bigquery.DatasetReference(project_id, tokyo_dataset), + "to_gbq_test", + ) ) assert table.num_rows > 0 def test_upload_data_tokyo_non_existing_dataset( self, project_id, random_dataset_id, bigquery_client ): + from google.cloud import bigquery + test_size = 10 df = make_mixed_dataframe_v2(test_size) non_existing_tokyo_dataset = random_dataset_id @@ -1490,8 +1568,11 @@ def test_upload_data_tokyo_non_existing_dataset( ) table = bigquery_client.get_table( - bigquery_client.dataset(non_existing_tokyo_dataset).table( - "to_gbq_test" + bigquery.TableReference( + bigquery.DatasetReference( + project_id, non_existing_tokyo_dataset + ), + "to_gbq_test", ) ) assert table.num_rows > 0 @@ -1500,9 +1581,15 @@ def test_upload_data_tokyo_non_existing_dataset( # _Dataset tests -def test_create_dataset(bigquery_client, gbq_dataset, random_dataset_id): +def test_create_dataset( + bigquery_client, gbq_dataset, random_dataset_id, project_id +): + from google.cloud import bigquery + gbq_dataset.create(random_dataset_id) - dataset_reference = bigquery_client.dataset(random_dataset_id) + dataset_reference = bigquery.DatasetReference( + project_id, random_dataset_id + ) assert bigquery_client.get_dataset(dataset_reference) is not None @@ -1593,8 +1680,8 @@ def test_verify_schema_allows_flexible_column_order(gbq_table, gbq_connector): } gbq_table.create(table_id, test_schema_1) - assert gbq_connector.verify_schema( - gbq_table.dataset_id, table_id, test_schema_2 + assert verify_schema( + gbq_connector, gbq_table.dataset_id, table_id, test_schema_2 ) @@ -1618,8 +1705,8 @@ def test_verify_schema_fails_different_data_type(gbq_table, gbq_connector): } gbq_table.create(table_id, test_schema_1) - assert not gbq_connector.verify_schema( - gbq_table.dataset_id, table_id, test_schema_2 + assert not verify_schema( + gbq_connector, gbq_table.dataset_id, table_id, test_schema_2 ) @@ -1643,8 +1730,8 @@ def test_verify_schema_fails_different_structure(gbq_table, gbq_connector): } gbq_table.create(table_id, test_schema_1) - assert not gbq_connector.verify_schema( - gbq_table.dataset_id, table_id, test_schema_2 + assert not verify_schema( + gbq_connector, gbq_table.dataset_id, table_id, test_schema_2 ) @@ -1668,8 +1755,8 @@ def test_verify_schema_ignores_field_mode(gbq_table, gbq_connector): } gbq_table.create(table_id, test_schema_1) - assert gbq_connector.verify_schema( - gbq_table.dataset_id, table_id, test_schema_2 + assert verify_schema( + gbq_connector, gbq_table.dataset_id, table_id, test_schema_2 ) @@ -1706,16 +1793,15 @@ def test_retrieve_schema(gbq_table, gbq_connector): } gbq_table.create(table_id, test_schema) - actual = pandas_gbq.schema._clean_schema_fields( - gbq_connector.schema(gbq_table.dataset_id, table_id) - ) expected = [ {"name": "A", "type": "FLOAT"}, {"name": "B", "type": "FLOAT"}, {"name": "C", "type": "STRING"}, {"name": "D", "type": "TIMESTAMP"}, ] - assert expected == actual, "Expected schema used to create table" + assert verify_schema( + gbq_connector, gbq_table.dataset_id, table_id, {"fields": expected} + ) def test_to_gbq_does_not_override_mode(gbq_table, gbq_connector): @@ -1752,5 +1838,6 @@ def test_to_gbq_does_not_override_mode(gbq_table, gbq_connector): if_exists="append", ) - actual = gbq_connector.schema(gbq_table.dataset_id, table_id) - assert table_schema["fields"] == actual + assert verify_schema( + gbq_connector, gbq_table.dataset_id, table_id, table_schema + ) From b4d3304d2b1b0aeaa42af7efc7a64326d6fa869e Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 19 Jan 2021 11:29:26 -0600 Subject: [PATCH 219/519] FIX: exclude google-cloud-bigquery==2.4.x from dependencies (#354) * FIX: exclude google-cloud-bigquery==2.4.x from dependencies * update changelog --- packages/pandas-gbq/docs/source/changelog.rst | 2 ++ packages/pandas-gbq/setup.py | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 372a0c61760e..2ca3650e0007 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -17,6 +17,8 @@ Dependencies ~~~~~~~~~~~~ - Drop support for Python 3.5 and 3.6. (:issue:`337`) +- Drop support for `google-cloud-bigquery==2.4.*` due to query hanging bug. + (:issue:`343`) .. _changelog-0.14.1: diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index 3ebe4a46cc73..a17c32e45cc7 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -22,7 +22,9 @@ def readme(): "pydata-google-auth", "google-auth", "google-auth-oauthlib", - "google-cloud-bigquery[bqstorage,pandas]>=1.11.1,<3.0.0dev", + # 2.4.* has a bug where waiting for the query can hang indefinitely. + # https://github.com/pydata/pandas-gbq/issues/343 + "google-cloud-bigquery[bqstorage,pandas]>=1.11.1,<3.0.0dev,!=2.4.*", ] extras = {"tqdm": "tqdm>=4.23.0"} From b02d001946113ba294f1b484bc585a1786d94a0f Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 30 Mar 2021 10:21:41 -0500 Subject: [PATCH 220/519] fix: avoid 403 from to_gbq when table has policyTags (#356) * fix: avoid 403 from to_gbq when table has policyTags * pin dependency versions in conda test session * remove pyarrow and bqstorage API from conda session with min pandas --- .../ci/requirements-3.7-0.23.2.conda | 3 +- packages/pandas-gbq/docs/source/changelog.rst | 5 + packages/pandas-gbq/pandas_gbq/features.py | 95 +++++++ packages/pandas-gbq/pandas_gbq/gbq.py | 100 ++------ packages/pandas-gbq/pandas_gbq/load.py | 48 ++-- packages/pandas-gbq/pandas_gbq/schema.py | 38 ++- .../pandas-gbq/tests/unit/test_features.py | 28 +++ packages/pandas-gbq/tests/unit/test_gbq.py | 235 +++++++----------- packages/pandas-gbq/tests/unit/test_load.py | 63 ++++- 9 files changed, 359 insertions(+), 256 deletions(-) create mode 100644 packages/pandas-gbq/pandas_gbq/features.py create mode 100644 packages/pandas-gbq/tests/unit/test_features.py diff --git a/packages/pandas-gbq/ci/requirements-3.7-0.23.2.conda b/packages/pandas-gbq/ci/requirements-3.7-0.23.2.conda index f36e096df9e6..af4768ab3405 100644 --- a/packages/pandas-gbq/ci/requirements-3.7-0.23.2.conda +++ b/packages/pandas-gbq/ci/requirements-3.7-0.23.2.conda @@ -2,8 +2,9 @@ codecov coverage fastavro flake8 +numpy==1.14.5 google-cloud-bigquery==1.11.1 -google-cloud-bigquery-storage pydata-google-auth pytest pytest-cov +tqdm==4.23.0 diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index 2ca3650e0007..e9553f88c5ee 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -13,6 +13,11 @@ Features client project. Specify the target table ID as ``project.dataset.table`` to use this feature. (:issue:`321`, :issue:`347`) +Bug fixes +~~~~~~~~~ + +- Avoid 403 error from ``to_gbq`` when table has ``policyTags``. (:issue:`354`) + Dependencies ~~~~~~~~~~~~ diff --git a/packages/pandas-gbq/pandas_gbq/features.py b/packages/pandas-gbq/pandas_gbq/features.py new file mode 100644 index 000000000000..3a63189b9ed8 --- /dev/null +++ b/packages/pandas-gbq/pandas_gbq/features.py @@ -0,0 +1,95 @@ +"""Module for checking dependency versions and supported features.""" + +# https://github.com/googleapis/python-bigquery/blob/master/CHANGELOG.md +BIGQUERY_MINIMUM_VERSION = "1.11.1" +BIGQUERY_CLIENT_INFO_VERSION = "1.12.0" +BIGQUERY_BQSTORAGE_VERSION = "1.24.0" +BIGQUERY_FROM_DATAFRAME_CSV_VERSION = "2.6.0" +PANDAS_VERBOSITY_DEPRECATION_VERSION = "0.23.0" + + +class Features: + def __init__(self): + self._bigquery_installed_version = None + self._pandas_installed_version = None + + @property + def bigquery_installed_version(self): + import google.cloud.bigquery + import pkg_resources + + if self._bigquery_installed_version is not None: + return self._bigquery_installed_version + + self._bigquery_installed_version = pkg_resources.parse_version( + google.cloud.bigquery.__version__ + ) + bigquery_minimum_version = pkg_resources.parse_version( + BIGQUERY_MINIMUM_VERSION + ) + + if self._bigquery_installed_version < bigquery_minimum_version: + raise ImportError( + "pandas-gbq requires google-cloud-bigquery >= {0}, " + "current version {1}".format( + bigquery_minimum_version, self._bigquery_installed_version + ) + ) + + return self._bigquery_installed_version + + @property + def bigquery_has_client_info(self): + import pkg_resources + + bigquery_client_info_version = pkg_resources.parse_version( + BIGQUERY_CLIENT_INFO_VERSION + ) + return self.bigquery_installed_version >= bigquery_client_info_version + + @property + def bigquery_has_bqstorage(self): + import pkg_resources + + bigquery_bqstorage_version = pkg_resources.parse_version( + BIGQUERY_BQSTORAGE_VERSION + ) + return self.bigquery_installed_version >= bigquery_bqstorage_version + + @property + def bigquery_has_from_dataframe_with_csv(self): + import pkg_resources + + bigquery_from_dataframe_version = pkg_resources.parse_version( + BIGQUERY_FROM_DATAFRAME_CSV_VERSION + ) + return ( + self.bigquery_installed_version >= bigquery_from_dataframe_version + ) + + @property + def pandas_installed_version(self): + import pandas + import pkg_resources + + if self._pandas_installed_version is not None: + return self._pandas_installed_version + + self._pandas_installed_version = pkg_resources.parse_version( + pandas.__version__ + ) + return self._pandas_installed_version + + @property + def pandas_has_deprecated_verbose(self): + import pkg_resources + + # Add check for Pandas version before showing deprecation warning. + # https://github.com/pydata/pandas-gbq/issues/157 + pandas_verbosity_deprecation = pkg_resources.parse_version( + PANDAS_VERBOSITY_DEPRECATION_VERSION + ) + return self.pandas_installed_version >= pandas_verbosity_deprecation + + +FEATURES = Features() diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 8d7c15c84b7c..884d5470cd09 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -16,100 +16,45 @@ from pandas_gbq.exceptions import AccessDenied from pandas_gbq.exceptions import PerformanceWarning +from pandas_gbq import features +from pandas_gbq.features import FEATURES import pandas_gbq.schema import pandas_gbq.timestamp logger = logging.getLogger(__name__) -BIGQUERY_INSTALLED_VERSION = None -BIGQUERY_CLIENT_INFO_VERSION = "1.12.0" -BIGQUERY_BQSTORAGE_VERSION = "1.24.0" -HAS_CLIENT_INFO = False -HAS_BQSTORAGE_SUPPORT = False - try: import tqdm # noqa except ImportError: tqdm = None -def _check_google_client_version(): - global BIGQUERY_INSTALLED_VERSION, HAS_CLIENT_INFO, HAS_BQSTORAGE_SUPPORT, SHOW_VERBOSE_DEPRECATION - - try: - import pkg_resources - - except ImportError: - raise ImportError("Could not import pkg_resources (setuptools).") - - # https://github.com/googleapis/python-bigquery/blob/master/CHANGELOG.md - bigquery_minimum_version = pkg_resources.parse_version("1.11.0") - bigquery_client_info_version = pkg_resources.parse_version( - BIGQUERY_CLIENT_INFO_VERSION - ) - bigquery_bqstorage_version = pkg_resources.parse_version( - BIGQUERY_BQSTORAGE_VERSION - ) - BIGQUERY_INSTALLED_VERSION = pkg_resources.get_distribution( - "google-cloud-bigquery" - ).parsed_version - - HAS_CLIENT_INFO = ( - BIGQUERY_INSTALLED_VERSION >= bigquery_client_info_version - ) - HAS_BQSTORAGE_SUPPORT = ( - BIGQUERY_INSTALLED_VERSION >= bigquery_bqstorage_version - ) - - if BIGQUERY_INSTALLED_VERSION < bigquery_minimum_version: - raise ImportError( - "pandas-gbq requires google-cloud-bigquery >= {0}, " - "current version {1}".format( - bigquery_minimum_version, BIGQUERY_INSTALLED_VERSION - ) - ) - - # Add check for Pandas version before showing deprecation warning. - # https://github.com/pydata/pandas-gbq/issues/157 - pandas_installed_version = pkg_resources.get_distribution( - "pandas" - ).parsed_version - pandas_version_wo_verbosity = pkg_resources.parse_version("0.23.0") - SHOW_VERBOSE_DEPRECATION = ( - pandas_installed_version >= pandas_version_wo_verbosity - ) - - def _test_google_api_imports(): + try: + import pkg_resources # noqa + except ImportError as ex: + raise ImportError("pandas-gbq requires setuptools") from ex try: import pydata_google_auth # noqa except ImportError as ex: - raise ImportError( - "pandas-gbq requires pydata-google-auth: {0}".format(ex) - ) + raise ImportError("pandas-gbq requires pydata-google-auth") from ex try: from google_auth_oauthlib.flow import InstalledAppFlow # noqa except ImportError as ex: - raise ImportError( - "pandas-gbq requires google-auth-oauthlib: {0}".format(ex) - ) + raise ImportError("pandas-gbq requires google-auth-oauthlib") from ex try: import google.auth # noqa except ImportError as ex: - raise ImportError("pandas-gbq requires google-auth: {0}".format(ex)) + raise ImportError("pandas-gbq requires google-auth") from ex try: from google.cloud import bigquery # noqa except ImportError as ex: - raise ImportError( - "pandas-gbq requires google-cloud-bigquery: {0}".format(ex) - ) - - _check_google_client_version() + raise ImportError("pandas-gbq requires google-cloud-bigquery") from ex class DatasetCreationError(ValueError): @@ -416,7 +361,7 @@ def get_client(self): # In addition to new enough version of google-api-core, a new enough # version of google-cloud-bigquery is required to populate the # client_info. - if HAS_CLIENT_INFO: + if FEATURES.bigquery_has_client_info: return bigquery.Client( project=self.project_id, credentials=self.credentials, @@ -550,14 +495,15 @@ def _download_results( if user_dtypes is None: user_dtypes = {} - if self.use_bqstorage_api and not HAS_BQSTORAGE_SUPPORT: + if self.use_bqstorage_api and not FEATURES.bigquery_has_bqstorage: warnings.warn( ( "use_bqstorage_api was set, but have google-cloud-bigquery " "version {}. Requires google-cloud-bigquery version " "{} or later." ).format( - BIGQUERY_INSTALLED_VERSION, BIGQUERY_BQSTORAGE_VERSION + FEATURES.bigquery_installed_version, + features.BIGQUERY_BQSTORAGE_VERSION, ), PerformanceWarning, stacklevel=4, @@ -568,7 +514,7 @@ def _download_results( create_bqstorage_client = False to_dataframe_kwargs = {} - if HAS_BQSTORAGE_SUPPORT: + if FEATURES.bigquery_has_bqstorage: to_dataframe_kwargs[ "create_bqstorage_client" ] = create_bqstorage_client @@ -880,7 +826,7 @@ def read_gbq( _test_google_api_imports() - if verbose is not None and SHOW_VERBOSE_DEPRECATION: + if verbose is not None and FEATURES.pandas_has_deprecated_verbose: warnings.warn( "verbose is deprecated and will be removed in " "a future version. Set logging level in order to vary " @@ -1054,7 +1000,7 @@ def to_gbq( _test_google_api_imports() - if verbose is not None and SHOW_VERBOSE_DEPRECATION: + if verbose is not None and FEATURES.pandas_has_deprecated_verbose: warnings.warn( "verbose is deprecated and will be removed in " "a future version. Set logging level in order to vary " @@ -1133,8 +1079,8 @@ def to_gbq( "schema of the destination table." ) - # Update the local `table_schema` so mode matches. - # See: https://github.com/pydata/pandas-gbq/issues/315 + # Update the local `table_schema` so mode (NULLABLE/REQUIRED) + # matches. See: https://github.com/pydata/pandas-gbq/issues/315 table_schema = pandas_gbq.schema.update_schema( table_schema, original_schema ) @@ -1252,7 +1198,6 @@ def create(self, table_id, schema): dataframe. """ from google.cloud.bigquery import DatasetReference - from google.cloud.bigquery import SchemaField from google.cloud.bigquery import Table from google.cloud.bigquery import TableReference @@ -1274,12 +1219,7 @@ def create(self, table_id, schema): DatasetReference(self.project_id, self.dataset_id), table_id ) table = Table(table_ref) - - schema = pandas_gbq.schema.add_default_nullable_mode(schema) - - table.schema = [ - SchemaField.from_api_repr(field) for field in schema["fields"] - ] + table.schema = pandas_gbq.schema.to_google_cloud_bigquery(schema) try: self.client.create_table(table) diff --git a/packages/pandas-gbq/pandas_gbq/load.py b/packages/pandas-gbq/pandas_gbq/load.py index d9e59c1d16eb..98211482e313 100644 --- a/packages/pandas-gbq/pandas_gbq/load.py +++ b/packages/pandas-gbq/pandas_gbq/load.py @@ -4,6 +4,7 @@ from google.cloud import bigquery +from pandas_gbq.features import FEATURES import pandas_gbq.schema @@ -30,10 +31,10 @@ def encode_chunk(dataframe): return io.BytesIO(body) -def encode_chunks(dataframe, chunksize=None): +def split_dataframe(dataframe, chunksize=None): dataframe = dataframe.reset_index(drop=True) if chunksize is None: - yield 0, encode_chunk(dataframe) + yield 0, dataframe return remaining_rows = len(dataframe) @@ -41,10 +42,10 @@ def encode_chunks(dataframe, chunksize=None): start_index = 0 while start_index < total_rows: end_index = start_index + chunksize - chunk_buffer = encode_chunk(dataframe[start_index:end_index]) + chunk = dataframe[start_index:end_index] start_index += chunksize remaining_rows = max(0, remaining_rows - chunksize) - yield remaining_rows, chunk_buffer + yield remaining_rows, chunk def load_chunks( @@ -60,24 +61,35 @@ def load_chunks( job_config.source_format = "CSV" job_config.allow_quoted_newlines = True - if schema is None: + # Explicit schema? Use that! + if schema is not None: + schema = pandas_gbq.schema.remove_policy_tags(schema) + job_config.schema = pandas_gbq.schema.to_google_cloud_bigquery(schema) + # If not, let BigQuery determine schema unless we are encoding the CSV files ourselves. + elif not FEATURES.bigquery_has_from_dataframe_with_csv: schema = pandas_gbq.schema.generate_bq_schema(dataframe) + schema = pandas_gbq.schema.remove_policy_tags(schema) + job_config.schema = pandas_gbq.schema.to_google_cloud_bigquery(schema) - schema = pandas_gbq.schema.add_default_nullable_mode(schema) + chunks = split_dataframe(dataframe, chunksize=chunksize) + for remaining_rows, chunk in chunks: + yield remaining_rows - job_config.schema = [ - bigquery.SchemaField.from_api_repr(field) for field in schema["fields"] - ] - - chunks = encode_chunks(dataframe, chunksize=chunksize) - for remaining_rows, chunk_buffer in chunks: - try: - yield remaining_rows - client.load_table_from_file( - chunk_buffer, + if FEATURES.bigquery_has_from_dataframe_with_csv: + client.load_table_from_dataframe( + chunk, destination_table_ref, job_config=job_config, location=location, ).result() - finally: - chunk_buffer.close() + else: + try: + chunk_buffer = encode_chunk(chunk) + client.load_table_from_file( + chunk_buffer, + destination_table_ref, + job_config=job_config, + location=location, + ).result() + finally: + chunk_buffer.close() diff --git a/packages/pandas-gbq/pandas_gbq/schema.py b/packages/pandas-gbq/pandas_gbq/schema.py index ffc1c36280cd..9deaeb7cf192 100644 --- a/packages/pandas-gbq/pandas_gbq/schema.py +++ b/packages/pandas-gbq/pandas_gbq/schema.py @@ -27,6 +27,19 @@ def to_pandas_gbq(client_schema): return {"fields": remote_fields} +def to_google_cloud_bigquery(pandas_gbq_schema): + """Given a schema in pandas-gbq API format, + return a sequence of :class:`google.cloud.bigquery.schema.SchemaField`. + """ + from google.cloud import bigquery + + # Need to convert from JSON representation to format used by client library. + schema = add_default_nullable_mode(pandas_gbq_schema) + return [ + bigquery.SchemaField.from_api_repr(field) for field in schema["fields"] + ] + + def _clean_schema_fields(fields): """Return a sanitized version of the schema for comparisons. @@ -129,13 +142,30 @@ def update_schema(schema_old, schema_new): def add_default_nullable_mode(schema): - """Manually create the schema objects, adding NULLABLE mode.""" - # Workaround for: - # https://github.com/GoogleCloudPlatform/google-cloud-python/issues/4456 - # + """Manually create the schema objects, adding NULLABLE mode. + + Workaround for error in SchemaField.from_api_repr, which required + "mode" to be set: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues/4456 + """ # Returns a copy rather than modifying the mutable arg, # per Issue #277 result = copy.deepcopy(schema) for field in result["fields"]: field.setdefault("mode", "NULLABLE") return result + + +def remove_policy_tags(schema): + """Manually create the schema objects, removing policyTags. + + Workaround for 403 error with policy tags, which are not required in a load + job: https://github.com/googleapis/python-bigquery/pull/557 + """ + # Returns a copy rather than modifying the mutable arg, + # per Issue #277 + result = copy.deepcopy(schema) + for field in result["fields"]: + if "policyTags" in field: + del field["policyTags"] + return result diff --git a/packages/pandas-gbq/tests/unit/test_features.py b/packages/pandas-gbq/tests/unit/test_features.py new file mode 100644 index 000000000000..65cefb1c50d9 --- /dev/null +++ b/packages/pandas-gbq/tests/unit/test_features.py @@ -0,0 +1,28 @@ +import pytest + +from pandas_gbq.features import FEATURES + + +@pytest.fixture(autouse=True) +def fresh_bigquery_version(monkeypatch): + monkeypatch.setattr(FEATURES, "_bigquery_installed_version", None) + + +@pytest.mark.parametrize( + ["bigquery_version", "expected"], + [ + ("1.11.1", False), + ("1.26.0", False), + ("2.5.4", False), + ("2.6.0", True), + ("2.6.1", True), + ("2.12.0", True), + ], +) +def test_bigquery_has_from_dataframe_with_csv( + monkeypatch, bigquery_version, expected +): + import google.cloud.bigquery + + monkeypatch.setattr(google.cloud.bigquery, "__version__", bigquery_version) + assert FEATURES.bigquery_has_from_dataframe_with_csv == expected diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index 4426e8dc84c9..fbf2150006b8 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -7,31 +7,21 @@ import numpy import pandas from pandas import DataFrame -import pkg_resources import pytest from pandas_gbq import gbq +from pandas_gbq.features import FEATURES pytestmark = pytest.mark.filterwarnings( "ignore:credentials from Google Cloud SDK" ) -pandas_installed_version = pkg_resources.get_distribution( - "pandas" -).parsed_version def _make_connector(project_id="some-project", **kwargs): return gbq.GbqConnector(project_id, **kwargs) -@pytest.fixture -def min_bq_version(): - import pkg_resources - - return pkg_resources.parse_version("1.11.0") - - def mock_get_credentials_no_project(*args, **kwargs): import google.auth.credentials @@ -101,7 +91,11 @@ def test__bqschema_to_nullsafe_dtypes(type_, expected): def test_GbqConnector_get_client_w_old_bq(monkeypatch, mock_bigquery_client): gbq._test_google_api_imports() connector = _make_connector() - monkeypatch.setattr(gbq, "HAS_CLIENT_INFO", False) + monkeypatch.setattr( + type(FEATURES), + "bigquery_has_client_info", + mock.PropertyMock(return_value=False), + ) connector.get_client() @@ -113,9 +107,8 @@ def test_GbqConnector_get_client_w_old_bq(monkeypatch, mock_bigquery_client): def test_GbqConnector_get_client_w_new_bq(mock_bigquery_client): gbq._test_google_api_imports() - pytest.importorskip( - "google.cloud.bigquery", minversion=gbq.BIGQUERY_CLIENT_INFO_VERSION - ) + if not FEATURES.bigquery_has_client_info: + pytest.skip("google-cloud-bigquery missing client_info feature") pytest.importorskip("google.api_core.client_info") connector = _make_connector() @@ -143,83 +136,58 @@ def test_to_gbq_with_no_project_id_given_should_fail(monkeypatch): gbq.to_gbq(DataFrame([[1]]), "dataset.tablename") -def test_to_gbq_with_verbose_new_pandas_warns_deprecation(min_bq_version): - import pkg_resources - - pandas_version = pkg_resources.parse_version("0.23.0") - with pytest.warns(FutureWarning), mock.patch( - "pkg_resources.Distribution.parsed_version", - new_callable=mock.PropertyMock, - ) as mock_version: - mock_version.side_effect = [min_bq_version, pandas_version] - try: - gbq.to_gbq( - DataFrame([[1]]), - "dataset.tablename", - project_id="my-project", - verbose=True, - ) - except gbq.TableCreationError: - pass - - -def test_to_gbq_with_not_verbose_new_pandas_warns_deprecation(min_bq_version): - import pkg_resources - - pandas_version = pkg_resources.parse_version("0.23.0") - with pytest.warns(FutureWarning), mock.patch( - "pkg_resources.Distribution.parsed_version", - new_callable=mock.PropertyMock, - ) as mock_version: - mock_version.side_effect = [min_bq_version, pandas_version] +@pytest.mark.parametrize(["verbose"], [(True,), (False,)]) +def test_to_gbq_with_verbose_new_pandas_warns_deprecation( + monkeypatch, verbose +): + monkeypatch.setattr( + type(FEATURES), + "pandas_has_deprecated_verbose", + mock.PropertyMock(return_value=True), + ) + with pytest.warns(FutureWarning, match="verbose is deprecated"): try: gbq.to_gbq( DataFrame([[1]]), "dataset.tablename", project_id="my-project", - verbose=False, + verbose=verbose, ) except gbq.TableCreationError: pass -def test_to_gbq_wo_verbose_w_new_pandas_no_warnings(recwarn, min_bq_version): - import pkg_resources - - pandas_version = pkg_resources.parse_version("0.23.0") - with mock.patch( - "pkg_resources.Distribution.parsed_version", - new_callable=mock.PropertyMock, - ) as mock_version: - mock_version.side_effect = [min_bq_version, pandas_version] - try: - gbq.to_gbq( - DataFrame([[1]]), "dataset.tablename", project_id="my-project" - ) - except gbq.TableCreationError: - pass - assert len(recwarn) == 0 - +def test_to_gbq_wo_verbose_w_new_pandas_no_warnings(monkeypatch, recwarn): + monkeypatch.setattr( + type(FEATURES), + "pandas_has_deprecated_verbose", + mock.PropertyMock(return_value=True), + ) + try: + gbq.to_gbq( + DataFrame([[1]]), "dataset.tablename", project_id="my-project" + ) + except gbq.TableCreationError: + pass + assert len(recwarn) == 0 -def test_to_gbq_with_verbose_old_pandas_no_warnings(recwarn, min_bq_version): - import pkg_resources - pandas_version = pkg_resources.parse_version("0.22.0") - with mock.patch( - "pkg_resources.Distribution.parsed_version", - new_callable=mock.PropertyMock, - ) as mock_version: - mock_version.side_effect = [min_bq_version, pandas_version] - try: - gbq.to_gbq( - DataFrame([[1]]), - "dataset.tablename", - project_id="my-project", - verbose=True, - ) - except gbq.TableCreationError: - pass - assert len(recwarn) == 0 +def test_to_gbq_with_verbose_old_pandas_no_warnings(monkeypatch, recwarn): + monkeypatch.setattr( + type(FEATURES), + "pandas_has_deprecated_verbose", + mock.PropertyMock(return_value=False), + ) + try: + gbq.to_gbq( + DataFrame([[1]]), + "dataset.tablename", + project_id="my-project", + verbose=True, + ) + except gbq.TableCreationError: + pass + assert len(recwarn) == 0 def test_to_gbq_with_private_key_raises_notimplementederror(): @@ -232,9 +200,7 @@ def test_to_gbq_with_private_key_raises_notimplementederror(): ) -def test_to_gbq_doesnt_run_query( - recwarn, mock_bigquery_client, min_bq_version -): +def test_to_gbq_doesnt_run_query(mock_bigquery_client): try: gbq.to_gbq( DataFrame([[1]]), "dataset.tablename", project_id="my-project" @@ -370,76 +336,54 @@ def test_read_gbq_with_max_results_ten(monkeypatch, mock_bigquery_client): mock_bigquery_client.list_rows.assert_called_with(mock.ANY, max_results=10) -def test_read_gbq_with_verbose_new_pandas_warns_deprecation(min_bq_version): - import pkg_resources - - pandas_version = pkg_resources.parse_version("0.23.0") - with pytest.warns(FutureWarning), mock.patch( - "pkg_resources.Distribution.parsed_version", - new_callable=mock.PropertyMock, - ) as mock_version: - mock_version.side_effect = [min_bq_version, pandas_version] - gbq.read_gbq("SELECT 1", project_id="my-project", verbose=True) +@pytest.mark.parametrize(["verbose"], [(True,), (False,)]) +def test_read_gbq_with_verbose_new_pandas_warns_deprecation( + monkeypatch, verbose +): + monkeypatch.setattr( + type(FEATURES), + "pandas_has_deprecated_verbose", + mock.PropertyMock(return_value=True), + ) + with pytest.warns(FutureWarning, match="verbose is deprecated"): + gbq.read_gbq("SELECT 1", project_id="my-project", verbose=verbose) -def test_read_gbq_with_not_verbose_new_pandas_warns_deprecation( - min_bq_version, -): - import pkg_resources - - pandas_version = pkg_resources.parse_version("0.23.0") - with pytest.warns(FutureWarning), mock.patch( - "pkg_resources.Distribution.parsed_version", - new_callable=mock.PropertyMock, - ) as mock_version: - mock_version.side_effect = [min_bq_version, pandas_version] - gbq.read_gbq("SELECT 1", project_id="my-project", verbose=False) - - -def test_read_gbq_wo_verbose_w_new_pandas_no_warnings(recwarn, min_bq_version): - import pkg_resources - - pandas_version = pkg_resources.parse_version("0.23.0") - with mock.patch( - "pkg_resources.Distribution.parsed_version", - new_callable=mock.PropertyMock, - ) as mock_version: - mock_version.side_effect = [min_bq_version, pandas_version] - gbq.read_gbq("SELECT 1", project_id="my-project", dialect="standard") - assert len(recwarn) == 0 - - -def test_read_gbq_with_old_bq_raises_importerror(): - import pkg_resources - - bigquery_version = pkg_resources.parse_version("0.27.0") - with pytest.raises(ImportError, match="google-cloud-bigquery"), mock.patch( - "pkg_resources.Distribution.parsed_version", - new_callable=mock.PropertyMock, - ) as mock_version: - mock_version.side_effect = [bigquery_version] - gbq.read_gbq( - "SELECT 1", - project_id="my-project", - ) +def test_read_gbq_wo_verbose_w_new_pandas_no_warnings(monkeypatch, recwarn): + monkeypatch.setattr( + type(FEATURES), + "pandas_has_deprecated_verbose", + mock.PropertyMock(return_value=False), + ) + gbq.read_gbq("SELECT 1", project_id="my-project", dialect="standard") + assert len(recwarn) == 0 -def test_read_gbq_with_verbose_old_pandas_no_warnings(recwarn, min_bq_version): - import pkg_resources +def test_read_gbq_with_old_bq_raises_importerror(monkeypatch): + import google.cloud.bigquery - pandas_version = pkg_resources.parse_version("0.22.0") - with mock.patch( - "pkg_resources.Distribution.parsed_version", - new_callable=mock.PropertyMock, - ) as mock_version: - mock_version.side_effect = [min_bq_version, pandas_version] + monkeypatch.setattr(google.cloud.bigquery, "__version__", "0.27.0") + monkeypatch.setattr(FEATURES, "_bigquery_installed_version", None) + with pytest.raises(ImportError, match="google-cloud-bigquery"): gbq.read_gbq( "SELECT 1", project_id="my-project", - dialect="standard", - verbose=True, ) - assert len(recwarn) == 0 + + +def test_read_gbq_with_verbose_old_pandas_no_warnings(monkeypatch, recwarn): + monkeypatch.setattr( + type(FEATURES), + "pandas_has_deprecated_verbose", + mock.PropertyMock(return_value=False), + ) + gbq.read_gbq( + "SELECT 1", + project_id="my-project", + dialect="standard", + verbose=True, + ) + assert len(recwarn) == 0 def test_read_gbq_with_private_raises_notimplmentederror(): @@ -542,8 +486,7 @@ def test_read_gbq_passes_dtypes( def test_read_gbq_use_bqstorage_api( mock_bigquery_client, mock_service_account_credentials ): - gbq._check_google_client_version() - if not gbq.HAS_BQSTORAGE_SUPPORT: + if not FEATURES.bigquery_has_bqstorage: pytest.skip("requires BigQuery Storage API") mock_service_account_credentials.project_id = "service_account_project_id" diff --git a/packages/pandas-gbq/tests/unit/test_load.py b/packages/pandas-gbq/tests/unit/test_load.py index 7ed463c18850..a864d972ddfb 100644 --- a/packages/pandas-gbq/tests/unit/test_load.py +++ b/packages/pandas-gbq/tests/unit/test_load.py @@ -2,13 +2,22 @@ import textwrap from io import StringIO +from unittest import mock import numpy import pandas +import pytest +from pandas_gbq.features import FEATURES from pandas_gbq import load +def load_method(bqclient): + if FEATURES.bigquery_has_from_dataframe_with_csv: + return bqclient.load_table_from_dataframe + return bqclient.load_table_from_file + + def test_encode_chunk_with_unicode(): """Test that a dataframe containing unicode can be encoded as a file. @@ -64,19 +73,59 @@ def test_encode_chunk_with_newlines(): assert '"ij\r\nkl"' in csv_string -def test_encode_chunks_splits_dataframe(): +def test_split_dataframe(): df = pandas.DataFrame(numpy.random.randn(6, 4), index=range(6)) - chunks = list(load.encode_chunks(df, chunksize=2)) + chunks = list(load.split_dataframe(df, chunksize=2)) assert len(chunks) == 3 - remaining, buffer = chunks[0] + remaining, chunk = chunks[0] assert remaining == 4 - assert len(buffer.readlines()) == 2 + assert len(chunk.index) == 2 def test_encode_chunks_with_chunksize_none(): df = pandas.DataFrame(numpy.random.randn(6, 4), index=range(6)) - chunks = list(load.encode_chunks(df)) + chunks = list(load.split_dataframe(df)) assert len(chunks) == 1 - remaining, buffer = chunks[0] + remaining, chunk = chunks[0] assert remaining == 0 - assert len(buffer.readlines()) == 6 + assert len(chunk.index) == 6 + + +@pytest.mark.parametrize( + ["bigquery_has_from_dataframe_with_csv"], [(True,), (False,)] +) +def test_load_chunks_omits_policy_tags( + monkeypatch, mock_bigquery_client, bigquery_has_from_dataframe_with_csv +): + """Ensure that policyTags are omitted. + + We don't want to change the policyTags via a load job, as this can cause + 403 error. See: https://github.com/googleapis/python-bigquery/pull/557 + """ + import google.cloud.bigquery + + monkeypatch.setattr( + type(FEATURES), + "bigquery_has_from_dataframe_with_csv", + mock.PropertyMock(return_value=bigquery_has_from_dataframe_with_csv), + ) + df = pandas.DataFrame({"col1": [1, 2, 3]}) + destination = google.cloud.bigquery.TableReference.from_string( + "my-project.my_dataset.my_table" + ) + schema = { + "fields": [ + {"name": "col1", "type": "INT64", "policyTags": ["tag1", "tag2"]} + ] + } + + _ = list( + load.load_chunks(mock_bigquery_client, df, destination, schema=schema) + ) + + mock_load = load_method(mock_bigquery_client) + assert mock_load.called + _, kwargs = mock_load.call_args + assert "job_config" in kwargs + sent_field = kwargs["job_config"].schema[0].to_api_repr() + assert "policyTags" not in sent_field From e46ec651569fe79f0c3a98f0d96075d34ebb07a4 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 30 Mar 2021 11:47:16 -0500 Subject: [PATCH 221/519] chore: release 0.15.0 (#359) --- packages/pandas-gbq/docs/source/changelog.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst index e9553f88c5ee..6af3af751f0e 100644 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ b/packages/pandas-gbq/docs/source/changelog.rst @@ -3,8 +3,8 @@ Changelog .. _changelog-0.15.0: -0.15.0 / TBD ------------- +0.15.0 / 2021-03-30 +------------------- Features ~~~~~~~~ @@ -12,11 +12,14 @@ Features - Load DataFrame with ``to_gbq`` to a table in a project different from the API client project. Specify the target table ID as ``project.dataset.table`` to use this feature. (:issue:`321`, :issue:`347`) +- Allow billing project to be separate from destination table project in + ``to_gbq``. (:issue:`321`) Bug fixes ~~~~~~~~~ - Avoid 403 error from ``to_gbq`` when table has ``policyTags``. (:issue:`354`) +- Avoid ``client.dataset`` deprecation warnings. (:issue:`312`) Dependencies ~~~~~~~~~~~~ From 0f470a8dbda31f4ba2c704705a69f1e027834d35 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 30 Jun 2021 17:41:22 +0000 Subject: [PATCH 222/519] chore: add license headers ~/go/bin/addlicense -c "pandas-gbq Authors" -l bsd -y 2017 . --- packages/pandas-gbq/.circleci/config.yml | 4 ++++ packages/pandas-gbq/.readthedocs.yml | 4 ++++ packages/pandas-gbq/.stickler.yml | 4 ++++ packages/pandas-gbq/benchmark/read_gbq_large_results.py | 4 ++++ packages/pandas-gbq/benchmark/read_gbq_small_results.py | 4 ++++ packages/pandas-gbq/ci/config_auth.sh | 4 ++++ packages/pandas-gbq/ci/run_conda.sh | 4 ++++ packages/pandas-gbq/ci/run_pip.sh | 4 ++++ packages/pandas-gbq/ci/run_tests.sh | 4 ++++ packages/pandas-gbq/codecov.yml | 4 ++++ packages/pandas-gbq/conftest.py | 4 ++++ packages/pandas-gbq/docs/source/_static/style.css | 6 ++++++ packages/pandas-gbq/docs/source/_templates/layout.html | 6 ++++++ packages/pandas-gbq/docs/source/conf.py | 4 ++++ packages/pandas-gbq/noxfile.py | 4 ++++ packages/pandas-gbq/pandas_gbq/__init__.py | 4 ++++ packages/pandas-gbq/pandas_gbq/_version.py | 4 ++++ packages/pandas-gbq/pandas_gbq/auth.py | 4 ++++ packages/pandas-gbq/pandas_gbq/exceptions.py | 4 ++++ packages/pandas-gbq/pandas_gbq/features.py | 4 ++++ packages/pandas-gbq/pandas_gbq/gbq.py | 4 ++++ packages/pandas-gbq/pandas_gbq/load.py | 4 ++++ packages/pandas-gbq/pandas_gbq/schema.py | 4 ++++ packages/pandas-gbq/pandas_gbq/timestamp.py | 4 ++++ packages/pandas-gbq/samples/__init__.py | 4 ++++ packages/pandas-gbq/samples/tests/__init__.py | 4 ++++ packages/pandas-gbq/setup.py | 4 ++++ packages/pandas-gbq/tests/__init__.py | 4 ++++ packages/pandas-gbq/tests/system/__init__.py | 4 ++++ packages/pandas-gbq/tests/system/conftest.py | 4 ++++ packages/pandas-gbq/tests/system/test_auth.py | 4 ++++ packages/pandas-gbq/tests/system/test_gbq.py | 4 ++++ .../pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py | 4 ++++ packages/pandas-gbq/tests/system/test_to_gbq.py | 4 ++++ packages/pandas-gbq/tests/unit/__init__.py | 4 ++++ packages/pandas-gbq/tests/unit/conftest.py | 4 ++++ packages/pandas-gbq/tests/unit/test_auth.py | 4 ++++ packages/pandas-gbq/tests/unit/test_context.py | 4 ++++ packages/pandas-gbq/tests/unit/test_features.py | 4 ++++ packages/pandas-gbq/tests/unit/test_gbq.py | 4 ++++ packages/pandas-gbq/tests/unit/test_load.py | 4 ++++ packages/pandas-gbq/tests/unit/test_schema.py | 4 ++++ packages/pandas-gbq/tests/unit/test_timestamp.py | 4 ++++ packages/pandas-gbq/versioneer.py | 4 ++++ 44 files changed, 180 insertions(+) diff --git a/packages/pandas-gbq/.circleci/config.yml b/packages/pandas-gbq/.circleci/config.yml index 817b2e73f083..1f9c1a42ef5d 100644 --- a/packages/pandas-gbq/.circleci/config.yml +++ b/packages/pandas-gbq/.circleci/config.yml @@ -1,3 +1,7 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + version: 2 jobs: "lint": diff --git a/packages/pandas-gbq/.readthedocs.yml b/packages/pandas-gbq/.readthedocs.yml index d0bfff551102..420befc5fcf3 100644 --- a/packages/pandas-gbq/.readthedocs.yml +++ b/packages/pandas-gbq/.readthedocs.yml @@ -1,3 +1,7 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + requirements_file: docs/requirements-docs.txt build: image: latest diff --git a/packages/pandas-gbq/.stickler.yml b/packages/pandas-gbq/.stickler.yml index 6ef8c8c06bba..7bb34d25b67f 100644 --- a/packages/pandas-gbq/.stickler.yml +++ b/packages/pandas-gbq/.stickler.yml @@ -1,3 +1,7 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + linters: black: config: ./pyproject.toml diff --git a/packages/pandas-gbq/benchmark/read_gbq_large_results.py b/packages/pandas-gbq/benchmark/read_gbq_large_results.py index 98d9ff53abeb..6a1d68cece58 100644 --- a/packages/pandas-gbq/benchmark/read_gbq_large_results.py +++ b/packages/pandas-gbq/benchmark/read_gbq_large_results.py @@ -1,3 +1,7 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + import pandas_gbq # Select 163 MB worth of data, to time how long it takes to download large diff --git a/packages/pandas-gbq/benchmark/read_gbq_small_results.py b/packages/pandas-gbq/benchmark/read_gbq_small_results.py index 8e91b0a0cfd0..f90d3c7b2c30 100644 --- a/packages/pandas-gbq/benchmark/read_gbq_small_results.py +++ b/packages/pandas-gbq/benchmark/read_gbq_small_results.py @@ -1,3 +1,7 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + import pandas_gbq # Select a few KB worth of data, to time downloading small result sets. diff --git a/packages/pandas-gbq/ci/config_auth.sh b/packages/pandas-gbq/ci/config_auth.sh index b6e46295dff9..cde115c72d29 100755 --- a/packages/pandas-gbq/ci/config_auth.sh +++ b/packages/pandas-gbq/ci/config_auth.sh @@ -1,4 +1,8 @@ #!/bin/bash +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + set -e # Don't set -x, because we don't want to leak keys. DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" diff --git a/packages/pandas-gbq/ci/run_conda.sh b/packages/pandas-gbq/ci/run_conda.sh index 526e6de630eb..e29da98aff86 100755 --- a/packages/pandas-gbq/ci/run_conda.sh +++ b/packages/pandas-gbq/ci/run_conda.sh @@ -1,4 +1,8 @@ #!/bin/bash +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + set -e -x DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" diff --git a/packages/pandas-gbq/ci/run_pip.sh b/packages/pandas-gbq/ci/run_pip.sh index 08628d888ca8..855b322e796d 100755 --- a/packages/pandas-gbq/ci/run_pip.sh +++ b/packages/pandas-gbq/ci/run_pip.sh @@ -1,4 +1,8 @@ #!/bin/bash +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + set -e -x DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" diff --git a/packages/pandas-gbq/ci/run_tests.sh b/packages/pandas-gbq/ci/run_tests.sh index 09a00f7cba03..3b0113bacdbd 100755 --- a/packages/pandas-gbq/ci/run_tests.sh +++ b/packages/pandas-gbq/ci/run_tests.sh @@ -1,4 +1,8 @@ #!/bin/bash +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + set -e -x DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" diff --git a/packages/pandas-gbq/codecov.yml b/packages/pandas-gbq/codecov.yml index 89cb6807a59d..4c2ed9b16e4a 100644 --- a/packages/pandas-gbq/codecov.yml +++ b/packages/pandas-gbq/codecov.yml @@ -1,3 +1,7 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + coverage: status: project: diff --git a/packages/pandas-gbq/conftest.py b/packages/pandas-gbq/conftest.py index b5803f374303..ed6ebb0fc5ad 100644 --- a/packages/pandas-gbq/conftest.py +++ b/packages/pandas-gbq/conftest.py @@ -1,3 +1,7 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + """Shared pytest fixtures for `tests/system` and `samples/tests` tests.""" import os diff --git a/packages/pandas-gbq/docs/source/_static/style.css b/packages/pandas-gbq/docs/source/_static/style.css index 7f69caf30b48..59a24072f882 100644 --- a/packages/pandas-gbq/docs/source/_static/style.css +++ b/packages/pandas-gbq/docs/source/_static/style.css @@ -1,3 +1,9 @@ +/** + * Copyright (c) 2017 pandas-gbq Authors All rights reserved. + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file. + */ + @import url("theme.css"); a.internal em {font-style: normal} diff --git a/packages/pandas-gbq/docs/source/_templates/layout.html b/packages/pandas-gbq/docs/source/_templates/layout.html index 4c57ba83056d..74f6910cf8fe 100644 --- a/packages/pandas-gbq/docs/source/_templates/layout.html +++ b/packages/pandas-gbq/docs/source/_templates/layout.html @@ -1,2 +1,8 @@ + + {% extends "!layout.html" %} {% set css_files = css_files + ["_static/style.css"] %} diff --git a/packages/pandas-gbq/docs/source/conf.py b/packages/pandas-gbq/docs/source/conf.py index fad0ca018198..afad588db751 100644 --- a/packages/pandas-gbq/docs/source/conf.py +++ b/packages/pandas-gbq/docs/source/conf.py @@ -1,3 +1,7 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + # -*- coding: utf-8 -*- # # pandas-gbq documentation build configuration file, created by diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 0af7c2f6b9b4..e1564138fc38 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -1,3 +1,7 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + """Nox test automation configuration. See: https://nox.readthedocs.io/en/latest/ diff --git a/packages/pandas-gbq/pandas_gbq/__init__.py b/packages/pandas-gbq/pandas_gbq/__init__.py index 401d114a4761..33f08cf42ba9 100644 --- a/packages/pandas-gbq/pandas_gbq/__init__.py +++ b/packages/pandas-gbq/pandas_gbq/__init__.py @@ -1,3 +1,7 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + from .gbq import to_gbq, read_gbq, Context, context # noqa from ._version import get_versions diff --git a/packages/pandas-gbq/pandas_gbq/_version.py b/packages/pandas-gbq/pandas_gbq/_version.py index 0183df38f84e..017eefdf9694 100644 --- a/packages/pandas-gbq/pandas_gbq/_version.py +++ b/packages/pandas-gbq/pandas_gbq/_version.py @@ -1,3 +1,7 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag diff --git a/packages/pandas-gbq/pandas_gbq/auth.py b/packages/pandas-gbq/pandas_gbq/auth.py index 27ecc861fc87..5215378737cd 100644 --- a/packages/pandas-gbq/pandas_gbq/auth.py +++ b/packages/pandas-gbq/pandas_gbq/auth.py @@ -1,3 +1,7 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + """Private module for fetching Google BigQuery credentials.""" import logging diff --git a/packages/pandas-gbq/pandas_gbq/exceptions.py b/packages/pandas-gbq/pandas_gbq/exceptions.py index dde45081bc8f..aa83cdd3c34d 100644 --- a/packages/pandas-gbq/pandas_gbq/exceptions.py +++ b/packages/pandas-gbq/pandas_gbq/exceptions.py @@ -1,3 +1,7 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + class AccessDenied(ValueError): """ Raised when invalid credentials are provided, or tokens have expired. diff --git a/packages/pandas-gbq/pandas_gbq/features.py b/packages/pandas-gbq/pandas_gbq/features.py index 3a63189b9ed8..5a90caa233c3 100644 --- a/packages/pandas-gbq/pandas_gbq/features.py +++ b/packages/pandas-gbq/pandas_gbq/features.py @@ -1,3 +1,7 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + """Module for checking dependency versions and supported features.""" # https://github.com/googleapis/python-bigquery/blob/master/CHANGELOG.md diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 884d5470cd09..e7f8b0ae4ed0 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -1,3 +1,7 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + import logging import time import warnings diff --git a/packages/pandas-gbq/pandas_gbq/load.py b/packages/pandas-gbq/pandas_gbq/load.py index 98211482e313..4eca7c560655 100644 --- a/packages/pandas-gbq/pandas_gbq/load.py +++ b/packages/pandas-gbq/pandas_gbq/load.py @@ -1,3 +1,7 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + """Helper methods for loading data into BigQuery""" import io diff --git a/packages/pandas-gbq/pandas_gbq/schema.py b/packages/pandas-gbq/pandas_gbq/schema.py index 9deaeb7cf192..ec81045c1559 100644 --- a/packages/pandas-gbq/pandas_gbq/schema.py +++ b/packages/pandas-gbq/pandas_gbq/schema.py @@ -1,3 +1,7 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + """Helper methods for BigQuery schemas""" import copy diff --git a/packages/pandas-gbq/pandas_gbq/timestamp.py b/packages/pandas-gbq/pandas_gbq/timestamp.py index 8bdcfa427205..e0b414759a02 100644 --- a/packages/pandas-gbq/pandas_gbq/timestamp.py +++ b/packages/pandas-gbq/pandas_gbq/timestamp.py @@ -1,3 +1,7 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + """Helpers for working with TIMESTAMP data type. Private module. diff --git a/packages/pandas-gbq/samples/__init__.py b/packages/pandas-gbq/samples/__init__.py index e69de29bb2d1..edbca6c3c489 100644 --- a/packages/pandas-gbq/samples/__init__.py +++ b/packages/pandas-gbq/samples/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + diff --git a/packages/pandas-gbq/samples/tests/__init__.py b/packages/pandas-gbq/samples/tests/__init__.py index e69de29bb2d1..edbca6c3c489 100644 --- a/packages/pandas-gbq/samples/tests/__init__.py +++ b/packages/pandas-gbq/samples/tests/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index a17c32e45cc7..224dc0076642 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -1,4 +1,8 @@ #!/usr/bin/env python +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + # -*- coding: utf-8 -*- import versioneer diff --git a/packages/pandas-gbq/tests/__init__.py b/packages/pandas-gbq/tests/__init__.py index e69de29bb2d1..edbca6c3c489 100644 --- a/packages/pandas-gbq/tests/__init__.py +++ b/packages/pandas-gbq/tests/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + diff --git a/packages/pandas-gbq/tests/system/__init__.py b/packages/pandas-gbq/tests/system/__init__.py index e69de29bb2d1..edbca6c3c489 100644 --- a/packages/pandas-gbq/tests/system/__init__.py +++ b/packages/pandas-gbq/tests/system/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + diff --git a/packages/pandas-gbq/tests/system/conftest.py b/packages/pandas-gbq/tests/system/conftest.py index 9d160b47aed0..a40ac47b4726 100644 --- a/packages/pandas-gbq/tests/system/conftest.py +++ b/packages/pandas-gbq/tests/system/conftest.py @@ -1,3 +1,7 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + import google.oauth2.service_account import pytest diff --git a/packages/pandas-gbq/tests/system/test_auth.py b/packages/pandas-gbq/tests/system/test_auth.py index eef18f96065a..5e8f5a4755af 100644 --- a/packages/pandas-gbq/tests/system/test_auth.py +++ b/packages/pandas-gbq/tests/system/test_auth.py @@ -1,3 +1,7 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + """System tests for fetching Google BigQuery credentials.""" import os diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index 037ff3a6b9f1..bbc0da642b2c 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -1,3 +1,7 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + # -*- coding: utf-8 -*- import datetime diff --git a/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py b/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py index 999667919c31..8b9c7ecc31e3 100644 --- a/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py +++ b/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py @@ -1,3 +1,7 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + """System tests for read_gbq using the BigQuery Storage API.""" import functools diff --git a/packages/pandas-gbq/tests/system/test_to_gbq.py b/packages/pandas-gbq/tests/system/test_to_gbq.py index ca5e406aa43c..59435c3351ef 100644 --- a/packages/pandas-gbq/tests/system/test_to_gbq.py +++ b/packages/pandas-gbq/tests/system/test_to_gbq.py @@ -1,3 +1,7 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + import functools import pandas import pandas.testing diff --git a/packages/pandas-gbq/tests/unit/__init__.py b/packages/pandas-gbq/tests/unit/__init__.py index e69de29bb2d1..edbca6c3c489 100644 --- a/packages/pandas-gbq/tests/unit/__init__.py +++ b/packages/pandas-gbq/tests/unit/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + diff --git a/packages/pandas-gbq/tests/unit/conftest.py b/packages/pandas-gbq/tests/unit/conftest.py index 490d5663f18b..cfa1e8199fca 100644 --- a/packages/pandas-gbq/tests/unit/conftest.py +++ b/packages/pandas-gbq/tests/unit/conftest.py @@ -1,3 +1,7 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + # -*- coding: utf-8 -*- from unittest import mock diff --git a/packages/pandas-gbq/tests/unit/test_auth.py b/packages/pandas-gbq/tests/unit/test_auth.py index 4f1e76d90c97..ca5447462c5c 100644 --- a/packages/pandas-gbq/tests/unit/test_auth.py +++ b/packages/pandas-gbq/tests/unit/test_auth.py @@ -1,3 +1,7 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + # -*- coding: utf-8 -*- import json diff --git a/packages/pandas-gbq/tests/unit/test_context.py b/packages/pandas-gbq/tests/unit/test_context.py index cb0961106329..c0521745f51c 100644 --- a/packages/pandas-gbq/tests/unit/test_context.py +++ b/packages/pandas-gbq/tests/unit/test_context.py @@ -1,3 +1,7 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + # -*- coding: utf-8 -*- from unittest import mock diff --git a/packages/pandas-gbq/tests/unit/test_features.py b/packages/pandas-gbq/tests/unit/test_features.py index 65cefb1c50d9..d1d5af815b54 100644 --- a/packages/pandas-gbq/tests/unit/test_features.py +++ b/packages/pandas-gbq/tests/unit/test_features.py @@ -1,3 +1,7 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + import pytest from pandas_gbq.features import FEATURES diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index fbf2150006b8..7476db3f37e2 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -1,3 +1,7 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + # -*- coding: utf-8 -*- import copy diff --git a/packages/pandas-gbq/tests/unit/test_load.py b/packages/pandas-gbq/tests/unit/test_load.py index a864d972ddfb..353b8bd122b8 100644 --- a/packages/pandas-gbq/tests/unit/test_load.py +++ b/packages/pandas-gbq/tests/unit/test_load.py @@ -1,3 +1,7 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + # -*- coding: utf-8 -*- import textwrap diff --git a/packages/pandas-gbq/tests/unit/test_schema.py b/packages/pandas-gbq/tests/unit/test_schema.py index 16286a2d412d..bd04508e1f7e 100644 --- a/packages/pandas-gbq/tests/unit/test_schema.py +++ b/packages/pandas-gbq/tests/unit/test_schema.py @@ -1,3 +1,7 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + import datetime import pandas diff --git a/packages/pandas-gbq/tests/unit/test_timestamp.py b/packages/pandas-gbq/tests/unit/test_timestamp.py index cdd0c55e0f26..6c9e32823a6b 100644 --- a/packages/pandas-gbq/tests/unit/test_timestamp.py +++ b/packages/pandas-gbq/tests/unit/test_timestamp.py @@ -1,3 +1,7 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + """Unit tests for TIMESTAMP data type helpers.""" import pandas diff --git a/packages/pandas-gbq/versioneer.py b/packages/pandas-gbq/versioneer.py index 0c6daedfdb44..2968e97bc824 100644 --- a/packages/pandas-gbq/versioneer.py +++ b/packages/pandas-gbq/versioneer.py @@ -1,3 +1,7 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + # Version: 0.18 From c6381c51cebbe5db35552fec2caffbab9e9b10b7 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 30 Jun 2021 17:51:16 +0000 Subject: [PATCH 223/519] add Google Code of Conduct and contributing policy --- packages/pandas-gbq/CODE_OF_CONDUCT.md | 94 ++++++++++++++++++++++++++ packages/pandas-gbq/CONTRIBUTING.md | 30 ++++++++ 2 files changed, 124 insertions(+) create mode 100644 packages/pandas-gbq/CODE_OF_CONDUCT.md diff --git a/packages/pandas-gbq/CODE_OF_CONDUCT.md b/packages/pandas-gbq/CODE_OF_CONDUCT.md new file mode 100644 index 000000000000..0bd87614b1cd --- /dev/null +++ b/packages/pandas-gbq/CODE_OF_CONDUCT.md @@ -0,0 +1,94 @@ +# Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of +experience, education, socio-economic status, nationality, personal appearance, +race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, or to ban temporarily or permanently any +contributor for other behaviors that they deem inappropriate, threatening, +offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +This Code of Conduct also applies outside the project spaces when the Project +Steward has a reasonable belief that an individual's behavior may have a +negative impact on the project or its community. + +## Conflict Resolution + +We do not believe that all conflict is bad; healthy debate and disagreement +often yield positive results. However, it is never okay to be disrespectful or +to engage in behavior that violates the project’s code of conduct. + +If you see someone violating the code of conduct, you are encouraged to address +the behavior directly with those involved. Many issues can be resolved quickly +and easily, and this gives people more control over the outcome of their +dispute. If you are unable to resolve the matter for any reason, or if the +behavior is threatening or harassing, report it. We are dedicated to providing +an environment where participants feel welcome and safe. + +Reports should be directed to *[PROJECT STEWARD NAME(s) AND EMAIL(s)]*, the +Project Steward(s) for *[PROJECT NAME]*. It is the Project Steward’s duty to +receive and address reported violations of the code of conduct. They will then +work with a committee consisting of representatives from the Open Source +Programs Office and the Google Open Source Strategy team. If for any reason you +are uncomfortable reaching out to the Project Steward, please email +opensource@google.com. + +We will investigate every complaint, but you may not receive a direct response. +We will use our discretion in determining when and how to follow up on reported +incidents, which may range from not taking action to permanent expulsion from +the project and project-sponsored spaces. We will notify the accused of the +report and provide them an opportunity to discuss it before any action is taken. +The identity of the reporter will be omitted from the details of the report +supplied to the accused. In potentially harmful situations, such as ongoing +harassment or threats to anyone's safety, we may take action without notice. + +## Attribution + +This Code of Conduct is adapted from the Contributor Covenant, version 1.4, +available at +https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + diff --git a/packages/pandas-gbq/CONTRIBUTING.md b/packages/pandas-gbq/CONTRIBUTING.md index e67468732a93..a4f779149628 100644 --- a/packages/pandas-gbq/CONTRIBUTING.md +++ b/packages/pandas-gbq/CONTRIBUTING.md @@ -1,5 +1,35 @@ # Contributing +We'd love to accept your patches and contributions to this project. There are +just a few small guidelines you need to follow. + +## Contributor License Agreement + +Contributions to this project must be accompanied by a Contributor License +Agreement (CLA). You (or your employer) retain the copyright to your +contribution; this simply gives us permission to use and redistribute your +contributions as part of the project. Head over to + to see your current agreements on file or +to sign a new one. + +You generally only need to submit a CLA once, so if you've already submitted one +(even if it was for a different project), you probably don't need to do it +again. + +## Code Reviews + +All submissions, including submissions by project members, require review. We +use GitHub pull requests for this purpose. Consult +[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more +information on using pull requests. + +## Community Guidelines + +This project follows +[Google's Open Source Community Guidelines](https://opensource.google/conduct/). + +## Developer Tips + See the [contributing guide in the pandas-gbq docs](http://pandas-gbq.readthedocs.io/en/latest/contributing.html). From 3823e27ab6e4aa5c276c54c95408a4b15a0d627b Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 16 Jul 2021 16:34:28 -0500 Subject: [PATCH 224/519] chore: add repository metadata (#370) --- packages/pandas-gbq/.repo-metadata.json | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 packages/pandas-gbq/.repo-metadata.json diff --git a/packages/pandas-gbq/.repo-metadata.json b/packages/pandas-gbq/.repo-metadata.json new file mode 100644 index 000000000000..72d7285adbff --- /dev/null +++ b/packages/pandas-gbq/.repo-metadata.json @@ -0,0 +1,13 @@ +{ + "name": "pandas-gbq", + "name_pretty": "Google BigQuery connector for pandas", + "product_documentation": "https://cloud.google.com/bigquery", + "client_documentation": "https://pandas-gbq.readthedocs.io/en/latest/", + "issue_tracker": "https://github.com/googleapis/python-bigquery-pandas/issues", + "release_level": "beta", + "language": "python", + "library_type": "INTEGRATION", + "repo": "googleapis/python-bigquery-pandas", + "distribution_name": "pandas-gbq", + "api_id": "bigquery.googleapis.com" + } From 3b0ded24d267c85914edf9fbf7e72f359bdeea07 Mon Sep 17 00:00:00 2001 From: "google-cloud-policy-bot[bot]" <80869356+google-cloud-policy-bot[bot]@users.noreply.github.com> Date: Wed, 21 Jul 2021 09:48:09 -0500 Subject: [PATCH 225/519] chore: add SECURITY.md (#373) Co-authored-by: google-cloud-policy-bot[bot] <80869356+google-cloud-policy-bot[bot]@users.noreply.github.com> --- packages/pandas-gbq/SECURITY.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 packages/pandas-gbq/SECURITY.md diff --git a/packages/pandas-gbq/SECURITY.md b/packages/pandas-gbq/SECURITY.md new file mode 100644 index 000000000000..8b58ae9c01ae --- /dev/null +++ b/packages/pandas-gbq/SECURITY.md @@ -0,0 +1,7 @@ +# Security Policy + +To report a security issue, please use [g.co/vulnz](https://g.co/vulnz). + +The Google Security Team will respond within 5 working days of your report on g.co/vulnz. + +We use g.co/vulnz for our intake, and do coordination and disclosure here using GitHub Security Advisory to privately discuss and fix the issue. From 9d4ea5f5b863bd73f86c469ff32ece27fca2c476 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Wed, 21 Jul 2021 17:20:00 +0200 Subject: [PATCH 226/519] chore: configure renovate (#369) Co-authored-by: Tim Swast --- packages/pandas-gbq/renovate.json | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 packages/pandas-gbq/renovate.json diff --git a/packages/pandas-gbq/renovate.json b/packages/pandas-gbq/renovate.json new file mode 100644 index 000000000000..f45d8f110c30 --- /dev/null +++ b/packages/pandas-gbq/renovate.json @@ -0,0 +1,5 @@ +{ + "extends": [ + "config:base" + ] +} From 8d65a5524ca9a6d1f61625e060b92b8701d34b7b Mon Sep 17 00:00:00 2001 From: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Date: Tue, 3 Aug 2021 08:20:42 -0600 Subject: [PATCH 227/519] chore: add CODEOWNERS (#374) OSPO's security recommendations include requiring CODEOWNERS review. --- packages/pandas-gbq/.github/CODEOWNERS | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 packages/pandas-gbq/.github/CODEOWNERS diff --git a/packages/pandas-gbq/.github/CODEOWNERS b/packages/pandas-gbq/.github/CODEOWNERS new file mode 100644 index 000000000000..3c9ab94c6c16 --- /dev/null +++ b/packages/pandas-gbq/.github/CODEOWNERS @@ -0,0 +1,11 @@ +# Code owners file. +# This file controls who is tagged for review for any given pull request. +# +# For syntax help see: +# https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners#codeowners-syntax + +# The @googleapis/yoshi-python is the default owner for changes in this repo +* @googleapis/api-bigquery @googleapis/yoshi-python + +# The python-samples-reviewers team is the default owner for samples changes +/samples/ @googleapis/api-bigquery @googleapis/python-samples-owners From 20f3d590f263be3265d6333d0b2cf4d8b7011ece Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 19 Aug 2021 08:32:47 -0500 Subject: [PATCH 228/519] docs: add markdown version of changelog (#372) * docs: add markdown version of changelog TODO: make changelog page a symlink and render markdown in sphinx * Use CHANGELOG.md from docs * remove one-time script --- .../.github/PULL_REQUEST_TEMPLATE.md | 1 - packages/pandas-gbq/CHANGELOG.md | 480 ++++++++++++++++++ .../pandas-gbq/docs/requirements-docs.txt | 1 + packages/pandas-gbq/docs/source/changelog.md | 1 + packages/pandas-gbq/docs/source/changelog.rst | 427 ---------------- packages/pandas-gbq/docs/source/conf.py | 3 +- .../pandas-gbq/docs/source/contributing.rst | 6 +- packages/pandas-gbq/docs/source/index.rst | 2 +- packages/pandas-gbq/release-procedure.md | 3 - 9 files changed, 487 insertions(+), 437 deletions(-) create mode 100644 packages/pandas-gbq/CHANGELOG.md create mode 120000 packages/pandas-gbq/docs/source/changelog.md delete mode 100644 packages/pandas-gbq/docs/source/changelog.rst diff --git a/packages/pandas-gbq/.github/PULL_REQUEST_TEMPLATE.md b/packages/pandas-gbq/.github/PULL_REQUEST_TEMPLATE.md index e434e5ea3faf..872eb0ff8405 100644 --- a/packages/pandas-gbq/.github/PULL_REQUEST_TEMPLATE.md +++ b/packages/pandas-gbq/.github/PULL_REQUEST_TEMPLATE.md @@ -1,4 +1,3 @@ - [ ] closes #xxxx - [ ] tests added / passed - [ ] passes `nox -s blacken lint` -- [ ] `docs/source/changelog.rst` entry \ No newline at end of file diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md new file mode 100644 index 000000000000..dd258c577e6a --- /dev/null +++ b/packages/pandas-gbq/CHANGELOG.md @@ -0,0 +1,480 @@ +# Changelog + +## 0.15.0 / 2021-03-30 + +### Features + +- Load DataFrame with `to_gbq` to a table in a project different from + the API client project. Specify the target table ID as + `project.dataset.table` to use this feature. + ([#321](https://github.com/googleapis/python-bigquery-pandas/issues/321), + [#347](https://github.com/googleapis/python-bigquery-pandas/issues/347)) +- Allow billing project to be separate from destination table project + in `to_gbq`. + ([#321](https://github.com/googleapis/python-bigquery-pandas/issues/321)) + +### Bug fixes + +- Avoid 403 error from `to_gbq` when table has `policyTags`. + ([#354](https://github.com/googleapis/python-bigquery-pandas/issues/354)) +- Avoid `client.dataset` deprecation warnings. + ([#312](https://github.com/googleapis/python-bigquery-pandas/issues/312)) + +### Dependencies + +- Drop support for Python 3.5 and 3.6. + ([#337](https://github.com/googleapis/python-bigquery-pandas/issues/337)) +- Drop support for google-cloud-bigquery==2.4.\* due to query + hanging bug. + ([#343](https://github.com/googleapis/python-bigquery-pandas/issues/343)) + +## 0.14.1 / 2020-11-10 + +### Bug fixes + +- Use `object` dtype for `TIME` columns. + ([#328](https://github.com/googleapis/python-bigquery-pandas/issues/328)) +- Encode floating point values with greater precision. + ([#326](https://github.com/googleapis/python-bigquery-pandas/issues/326)) +- Support `INT64` and other standard SQL aliases in + `~pandas_gbq.to_gbq` `table_schema` argument. + ([#322](https://github.com/googleapis/python-bigquery-pandas/issues/322)) + +## 0.14.0 / 2020-10-05 + +- Add `dtypes` argument to `read_gbq`. Use this argument to override + the default `dtype` for a particular column in the query results. + For example, this can be used to select nullable integer columns as + the `Int64` nullable integer pandas extension type. + ([#242](https://github.com/googleapis/python-bigquery-pandas/issues/242), + [#332](https://github.com/googleapis/python-bigquery-pandas/issues/332)) + +``` python +df = gbq.read_gbq( + "SELECT CAST(NULL AS INT64) AS null_integer", + dtypes={"null_integer": "Int64"}, +) +``` + +### Dependency updates + +- Support `google-cloud-bigquery-storage` 2.0 and higher. + ([#329](https://github.com/googleapis/python-bigquery-pandas/issues/329)) +- Update the minimum version of `pandas` to 0.20.1. + ([#331](https://github.com/googleapis/python-bigquery-pandas/issues/331)) + +### Internal changes + +- Update tests to run against Python 3.8. + ([#331](https://github.com/googleapis/python-bigquery-pandas/issues/331)) + +## 0.13.3 / 2020-09-30 + +- Include needed "extras" from `google-cloud-bigquery` package as + dependencies. Exclude incompatible 2.0 version. + ([#324](https://github.com/googleapis/python-bigquery-pandas/issues/324), + [#329](https://github.com/googleapis/python-bigquery-pandas/issues/329)) + +## 0.13.2 / 2020-05-14 + +- Fix `Provided Schema does not match Table` error when the existing + table contains required fields. + ([#315](https://github.com/googleapis/python-bigquery-pandas/issues/315)) + +## 0.13.1 / 2020-02-13 + +- Fix `AttributeError` with BQ Storage API to download empty results. + ([#299](https://github.com/googleapis/python-bigquery-pandas/issues/299)) + +## 0.13.0 / 2019-12-12 + +- Raise `NotImplementedError` when the deprecated `private_key` + argument is used. + ([#301](https://github.com/googleapis/python-bigquery-pandas/issues/301)) + +## 0.12.0 / 2019-11-25 + +### New features + +- Add `max_results` argument to `~pandas_gbq.read_gbq()`. Use this + argument to limit the number of rows in the results DataFrame. Set + `max_results` to 0 to ignore query outputs, such as for DML or DDL + queries. + ([#102](https://github.com/googleapis/python-bigquery-pandas/issues/102)) +- Add `progress_bar_type` argument to `~pandas_gbq.read_gbq()`. Use + this argument to display a progress bar when downloading data. + ([#182](https://github.com/googleapis/python-bigquery-pandas/issues/182)) + +### Bug fixes + +- Fix resource leak with `use_bqstorage_api` by closing BigQuery + Storage API client after use. + ([#294](https://github.com/googleapis/python-bigquery-pandas/issues/294)) + +### Dependency updates + +- Update the minimum version of `google-cloud-bigquery` to 1.11.1. + ([#296](https://github.com/googleapis/python-bigquery-pandas/issues/296)) + +### Documentation + +- Add code samples to introduction and refactor howto guides. + ([#239](https://github.com/googleapis/python-bigquery-pandas/issues/239)) + +## 0.11.0 / 2019-07-29 + +- **Breaking Change:** Python 2 support has been dropped. This is to + align with the pandas package which dropped Python 2 support at the + end of 2019. + ([#268](https://github.com/googleapis/python-bigquery-pandas/issues/268)) + +### Enhancements + +- Ensure `table_schema` argument is not modified inplace. + ([#278](https://github.com/googleapis/python-bigquery-pandas/issues/278)) + +### Implementation changes + +- Use object dtype for `STRING`, `ARRAY`, and `STRUCT` columns when + there are zero rows. + ([#285](https://github.com/googleapis/python-bigquery-pandas/issues/285)) + +### Internal changes + +- Populate `user-agent` with `pandas` version information. + ([#281](https://github.com/googleapis/python-bigquery-pandas/issues/281)) +- Fix `pytest.raises` usage for latest pytest. Fix warnings in tests. + ([#282](https://github.com/googleapis/python-bigquery-pandas/issues/282)) +- Update CI to install nightly packages in the conda tests. + ([#254](https://github.com/googleapis/python-bigquery-pandas/issues/254)) + +## 0.10.0 / 2019-04-05 + +- **Breaking Change:** Default SQL dialect is now `standard`. Use + `pandas_gbq.context.dialect` to override the default value. + ([#195](https://github.com/googleapis/python-bigquery-pandas/issues/195), + [#245](https://github.com/googleapis/python-bigquery-pandas/issues/245)) + +### Documentation + +- Document `BigQuery data type to pandas dtype conversion + ` for `read_gbq`. + ([#269](https://github.com/googleapis/python-bigquery-pandas/issues/269)) + +### Dependency updates + +- Update the minimum version of `google-cloud-bigquery` to 1.9.0. + ([#247](https://github.com/googleapis/python-bigquery-pandas/issues/247)) +- Update the minimum version of `pandas` to 0.19.0. + ([#262](https://github.com/googleapis/python-bigquery-pandas/issues/262)) + +### Internal changes + +- Update the authentication credentials. **Note:** You may need to set + `reauth=True` in order to update your credentials to the most recent + version. This is required to use new functionality such as the + BigQuery Storage API. + ([#267](https://github.com/googleapis/python-bigquery-pandas/issues/267)) +- Use `to_dataframe()` from `google-cloud-bigquery` in the + `read_gbq()` function. + ([#247](https://github.com/googleapis/python-bigquery-pandas/issues/247)) + +### Enhancements + +- Fix a bug where pandas-gbq could not upload an empty DataFrame. + ([#237](https://github.com/googleapis/python-bigquery-pandas/issues/237)) +- Allow `table_schema` in `to_gbq` to contain only a subset of + columns, with the rest being populated using the DataFrame dtypes + ([#218](https://github.com/googleapis/python-bigquery-pandas/issues/218)) + (contributed by @johnpaton) +- Read `project_id` in `to_gbq` from provided `credentials` if + available (contributed by @daureg) +- `read_gbq` uses the timezone-aware + `DatetimeTZDtype(unit='ns', tz='UTC')` dtype for BigQuery + `TIMESTAMP` columns. + ([#269](https://github.com/googleapis/python-bigquery-pandas/issues/269)) +- Add `use_bqstorage_api` to `read_gbq`. The BigQuery Storage API can + be used to download large query results (>125 MB) more quickly. If + the BQ Storage API can't be used, the BigQuery API is used instead. + ([#133](https://github.com/googleapis/python-bigquery-pandas/issues/133), + [#270](https://github.com/googleapis/python-bigquery-pandas/issues/270)) + +## 0.9.0 / 2019-01-11 + +- Warn when deprecated `private_key` parameter is used + ([#240](https://github.com/googleapis/python-bigquery-pandas/issues/240)) +- **New dependency** Use the `pydata-google-auth` package for + authentication. + ([#241](https://github.com/googleapis/python-bigquery-pandas/issues/241)) + +## 0.8.0 / 2018-11-12 + +### Breaking changes + +- **Deprecate** `private_key` parameter to `pandas_gbq.read_gbq` and + `pandas_gbq.to_gbq` in favor of new `credentials` argument. Instead, + create a credentials object using + `google.oauth2.service_account.Credentials.from_service_account_info` + or + `google.oauth2.service_account.Credentials.from_service_account_file`. + See the `authentication how-to guide ` for + examples. + ([#161](https://github.com/googleapis/python-bigquery-pandas/issues/161), + [#231](https://github.com/googleapis/python-bigquery-pandas/issues/231)) + +### Enhancements + +- Allow newlines in data passed to `to_gbq`. + ([#180](https://github.com/googleapis/python-bigquery-pandas/issues/180)) +- Add `pandas_gbq.context.dialect` to allow overriding the default SQL + syntax dialect. + ([#195](https://github.com/googleapis/python-bigquery-pandas/issues/195), + [#235](https://github.com/googleapis/python-bigquery-pandas/issues/235)) +- Support Python 3.7. + ([#197](https://github.com/googleapis/python-bigquery-pandas/issues/197), + [#232](https://github.com/googleapis/python-bigquery-pandas/issues/232)) + +### Internal changes + +- Migrate tests to CircleCI. + ([#228](https://github.com/googleapis/python-bigquery-pandas/issues/228), + [#232](https://github.com/googleapis/python-bigquery-pandas/issues/232)) + +## 0.7.0 / 2018-10-19 + +- int columns which contain NULL are now cast to float, rather than object type. + ([#174](https://github.com/googleapis/python-bigquery-pandas/issues/174)) +- DATE, DATETIME and TIMESTAMP columns are now parsed as pandas' + timestamp objects + ([#224](https://github.com/googleapis/python-bigquery-pandas/issues/224)) +- Add `pandas_gbq.Context` to cache credentials in-memory, across + calls to `read_gbq` and `to_gbq`. + ([#198](https://github.com/googleapis/python-bigquery-pandas/issues/198), + [#208](https://github.com/googleapis/python-bigquery-pandas/issues/208)) +- Fast queries now do not log above `DEBUG` level. + ([#204](https://github.com/googleapis/python-bigquery-pandas/issues/204)) + With BigQuery's release of + [clustering](https://cloud.google.com/bigquery/docs/clustered-tables) + querying smaller samples of data is now faster and cheaper. +- Don't load credentials from disk if reauth is `True`. + ([#212](https://github.com/googleapis/python-bigquery-pandas/issues/212)) + This fixes a bug where pandas-gbq could not refresh credentials if + the cached credentials were invalid, revoked, or expired, even when + `reauth=True`. +- Catch RefreshError when trying credentials. + ([#226](https://github.com/googleapis/python-bigquery-pandas/issues/226)) + +### Internal changes + +- Avoid listing datasets and tables in system tests. + ([#215](https://github.com/googleapis/python-bigquery-pandas/issues/215)) +- Improved performance from eliminating some duplicative parsing steps + ([#224](https://github.com/googleapis/python-bigquery-pandas/issues/224)) + +## 0.6.1 / 2018-09-11 + +- Improved `read_gbq` performance and memory consumption by delegating + `DataFrame` construction to the Pandas library, radically reducing + the number of loops that execute in python + ([#128](https://github.com/googleapis/python-bigquery-pandas/issues/128)) +- Reduced verbosity of logging from `read_gbq`, particularly for short + queries. + ([#201](https://github.com/googleapis/python-bigquery-pandas/issues/201)) +- Avoid `SELECT 1` query when running `to_gbq`. + ([#202](https://github.com/googleapis/python-bigquery-pandas/issues/202)) + +## 0.6.0 / 2018-08-15 + +- Warn when `dialect` is not passed in to `read_gbq`. The default + dialect will be changing from 'legacy' to 'standard' in a future + version. + ([#195](https://github.com/googleapis/python-bigquery-pandas/issues/195)) +- Use general float with 15 decimal digit precision when writing to + local CSV buffer in `to_gbq`. This prevents numerical overflow in + certain edge cases. + ([#192](https://github.com/googleapis/python-bigquery-pandas/issues/192)) + +## 0.5.0 / 2018-06-15 + +- Project ID parameter is optional in `read_gbq` and `to_gbq` when it + can inferred from the environment. Note: you must still pass in a + project ID when using user-based authentication. + ([#103](https://github.com/googleapis/python-bigquery-pandas/issues/103)) +- Progress bar added for `to_gbq`, through an optional library tqdm as dependency. + ([#162](https://github.com/googleapis/python-bigquery-pandas/issues/162)) +- Add location parameter to `read_gbq` and `to_gbq` so that pandas-gbq + can work with datasets in the Tokyo region. + ([#177](https://github.com/googleapis/python-bigquery-pandas/issues/177)) + +### Documentation + +- Add `authentication how-to guide `. + ([#183](https://github.com/googleapis/python-bigquery-pandas/issues/183)) +- Update `contributing` guide with new paths to tests. + ([#154](https://github.com/googleapis/python-bigquery-pandas/issues/154), + [#164](https://github.com/googleapis/python-bigquery-pandas/issues/164)) + +### Internal changes + +- Tests now use nox to run in multiple + Python environments. + ([#52](https://github.com/googleapis/python-bigquery-pandas/issues/52)) +- Renamed internal modules. + ([#154](https://github.com/googleapis/python-bigquery-pandas/issues/154)) +- Refactored auth to an internal auth module. + ([#176](https://github.com/googleapis/python-bigquery-pandas/issues/176)) +- Add unit tests for `get_credentials()`. + ([#184](https://github.com/googleapis/python-bigquery-pandas/issues/184)) + +## 0.4.1 / 2018-04-05 + +- Only show `verbose` deprecation warning if Pandas version does not + populate it. + ([#157](https://github.com/googleapis/python-bigquery-pandas/issues/157)) + +## 0.4.0 / 2018-04-03 + +- Fix bug in read_gbq when building a + dataframe with integer columns on Windows. Explicitly use 64bit + integers when converting from BQ types. + ([#119](https://github.com/googleapis/python-bigquery-pandas/issues/119)) +- Fix bug in read_gbq when querying for + an array of floats + ([#123](https://github.com/googleapis/python-bigquery-pandas/issues/123)) +- Fix bug in read_gbq with + configuration argument. Updates read_gbq to account for breaking change in + the way `google-cloud-python` version 0.32.0+ handles query + configuration API representation. + ([#152](https://github.com/googleapis/python-bigquery-pandas/issues/152)) +- Fix bug in to_gbq where seconds were + discarded in timestamp columns. + ([#148](https://github.com/googleapis/python-bigquery-pandas/issues/148)) +- Fix bug in to_gbq when supplying a + user-defined schema + ([#150](https://github.com/googleapis/python-bigquery-pandas/issues/150)) +- **Deprecate** the `verbose` parameter in read_gbq and to_gbq. Messages use the logging module + instead of printing progress directly to standard output. + ([#12](https://github.com/googleapis/python-bigquery-pandas/issues/12)) + +## 0.3.1 / 2018-02-13 + +- Fix an issue where Unicode couldn't be uploaded in Python 2 + ([#106](https://github.com/googleapis/python-bigquery-pandas/issues/106)) +- Add support for a passed schema in `` `to_gbq `` instead inferring the schema from the passed + DataFrame with DataFrame.dtypes + (#46 + \<\>\`\_) +- Fix an issue where a dataframe containing both integer and floating + point columns could not be uploaded with `to_gbq` + ([#116](https://github.com/googleapis/python-bigquery-pandas/issues/116)) +- `to_gbq` now uses `to_csv` to avoid manually looping over rows in a + dataframe (should result in faster table uploads) + ([#96](https://github.com/googleapis/python-bigquery-pandas/issues/96)) + +## 0.3.0 / 2018-01-03 + +- Use the + [google-cloud-bigquery](https://googlecloudplatform.github.io/google-cloud-python/latest/bigquery/usage.html) + library for API calls. The `google-cloud-bigquery` package is a new + dependency, and dependencies on `google-api-python-client` and + `httplib2` are removed. See the [installation + guide](https://pandas-gbq.readthedocs.io/en/latest/install.html#dependencies) + for more details. + ([#93](https://github.com/googleapis/python-bigquery-pandas/issues/93)) +- Structs and arrays are now named properly + ([#23](https://github.com/googleapis/python-bigquery-pandas/issues/23)) + and BigQuery functions like `array_agg` no longer run into errors + during type conversion + ([#22](https://github.com/googleapis/python-bigquery-pandas/issues/22)). +- `to_gbq` now uses a load job instead of the streaming API. Remove + `StreamingInsertError` class, as it is no longer used by `to_gbq`. + ([#7](https://github.com/googleapis/python-bigquery-pandas/issues/7), + [#75](https://github.com/googleapis/python-bigquery-pandas/issues/75)) + +## 0.2.1 / 2017-11-27 + +- `read_gbq` now raises `QueryTimeout` if the request exceeds the + `query.timeoutMs` value specified in the BigQuery configuration. + ([#76](https://github.com/googleapis/python-bigquery-pandas/issues/76)) +- Environment variable `PANDAS_GBQ_CREDENTIALS_FILE` can now be used + to override the default location where the BigQuery user account + credentials are stored. + ([#86](https://github.com/googleapis/python-bigquery-pandas/issues/86)) +- BigQuery user account credentials are now stored in an + application-specific hidden user folder on the operating system. + ([#41](https://github.com/googleapis/python-bigquery-pandas/issues/41)) + +## 0.2.0 / 2017-07-24 + +- Drop support for Python 3.4 + ([#40](https://github.com/googleapis/python-bigquery-pandas/issues/40)) +- The dataframe passed to + `` `.to_gbq(...., if_exists='append') `` + needs to contain only a subset of the fields in the BigQuery schema. + (#24 + \<\>\`\_) +- Use the [google-auth](https://google-auth.readthedocs.io/en/latest/) + library for authentication because `oauth2client` is deprecated. + ([#39](https://github.com/googleapis/python-bigquery-pandas/issues/39)) +- `read_gbq` now has a `auth_local_webserver` boolean argument for + controlling whether to use web server or console flow when getting + user credentials. Replaces --noauth_local_webserver command line + argument. + ([#35](https://github.com/googleapis/python-bigquery-pandas/issues/35)) +- `read_gbq` now displays the BigQuery Job ID and standard price in + verbose output. + ([#70](https://github.com/googleapis/python-bigquery-pandas/issues/70) + and + [#71](https://github.com/googleapis/python-bigquery-pandas/issues/71)) + +## 0.1.6 / 2017-05-03 + +- All gbq errors will simply be subclasses of `ValueError` and no + longer inherit from the deprecated `PandasError`. + +## 0.1.4 / 2017-03-17 + +- `InvalidIndexColumn` will be raised instead of `InvalidColumnOrder` + in `read_gbq` when the index column specified does not exist in the + BigQuery schema. + ([#6](https://github.com/googleapis/python-bigquery-pandas/issues/6)) + +## 0.1.3 / 2017-03-04 + +- Bug with appending to a BigQuery table where fields have modes + (NULLABLE,REQUIRED,REPEATED) specified. These modes were compared + versus the remote schema and writing a table via `to_gbq` would + previously raise. + ([#13](https://github.com/googleapis/python-bigquery-pandas/issues/13)) + +## 0.1.2 / 2017-02-23 + +Initial release of transfered code from +[pandas](https://github.com/pandas-dev/pandas) + +Includes patches since the 0.19.2 release on pandas with the following: + +- `read_gbq` now allows query configuration preferences + [pandas-GH#14742](https://github.com/pandas-dev/pandas/pull/14742) +- `read_gbq` now stores `INTEGER` columns as `dtype=object` if they + contain `NULL` values. Otherwise they are stored as `int64`. This + prevents precision lost for integers greather than 2\**53. + Furthermore \`\`FLOAT\`\` columns with values above 10*\*4 are no + longer casted to `int64` which also caused precision loss + [pandas-GH#14064](https://github.com/pandas-dev/pandas/pull/14064), + and + [pandas-GH#14305](https://github.com/pandas-dev/pandas/pull/14305) diff --git a/packages/pandas-gbq/docs/requirements-docs.txt b/packages/pandas-gbq/docs/requirements-docs.txt index afd31d061e10..af49a2462108 100644 --- a/packages/pandas-gbq/docs/requirements-docs.txt +++ b/packages/pandas-gbq/docs/requirements-docs.txt @@ -1,6 +1,7 @@ ipython matplotlib numpydoc +recommonmark sphinx sphinx_rtd_theme pandas diff --git a/packages/pandas-gbq/docs/source/changelog.md b/packages/pandas-gbq/docs/source/changelog.md new file mode 120000 index 000000000000..699cc9e7b7c5 --- /dev/null +++ b/packages/pandas-gbq/docs/source/changelog.md @@ -0,0 +1 @@ +../../CHANGELOG.md \ No newline at end of file diff --git a/packages/pandas-gbq/docs/source/changelog.rst b/packages/pandas-gbq/docs/source/changelog.rst deleted file mode 100644 index 6af3af751f0e..000000000000 --- a/packages/pandas-gbq/docs/source/changelog.rst +++ /dev/null @@ -1,427 +0,0 @@ -Changelog -========= - -.. _changelog-0.15.0: - -0.15.0 / 2021-03-30 -------------------- - -Features -~~~~~~~~ - -- Load DataFrame with ``to_gbq`` to a table in a project different from the API - client project. Specify the target table ID as ``project.dataset.table`` to - use this feature. (:issue:`321`, :issue:`347`) -- Allow billing project to be separate from destination table project in - ``to_gbq``. (:issue:`321`) - -Bug fixes -~~~~~~~~~ - -- Avoid 403 error from ``to_gbq`` when table has ``policyTags``. (:issue:`354`) -- Avoid ``client.dataset`` deprecation warnings. (:issue:`312`) - -Dependencies -~~~~~~~~~~~~ - -- Drop support for Python 3.5 and 3.6. (:issue:`337`) -- Drop support for `google-cloud-bigquery==2.4.*` due to query hanging bug. - (:issue:`343`) - - -.. _changelog-0.14.1: - -0.14.1 / 2020-11-10 -------------------- - -Bug fixes -~~~~~~~~~ - -- Use ``object`` dtype for ``TIME`` columns. (:issue:`328`) -- Encode floating point values with greater precision. (:issue:`326`) -- Support ``INT64`` and other standard SQL aliases in - :func:`~pandas_gbq.to_gbq` ``table_schema`` argument. (:issue:`322`) - - -.. _changelog-0.14.0: - -0.14.0 / 2020-10-05 -------------------- - -- Add ``dtypes`` argument to ``read_gbq``. Use this argument to override the - default ``dtype`` for a particular column in the query results. For - example, this can be used to select nullable integer columns as the - ``Int64`` nullable integer pandas extension type. (:issue:`242`, - :issue:`332`) - -.. code-block:: python - - df = gbq.read_gbq( - "SELECT CAST(NULL AS INT64) AS null_integer", - dtypes={"null_integer": "Int64"}, - ) - -Dependency updates -~~~~~~~~~~~~~~~~~~ - -- Support ``google-cloud-bigquery-storage`` 2.0 and higher. (:issue:`329`) -- Update the minimum version of ``pandas`` to 0.20.1. - (:issue:`331`) - -Internal changes -~~~~~~~~~~~~~~~~ - -- Update tests to run against Python 3.8. (:issue:`331`) - - -.. _changelog-0.13.3: - -0.13.3 / 2020-09-30 -------------------- - -- Include needed "extras" from ``google-cloud-bigquery`` package as - dependencies. Exclude incompatible 2.0 version. (:issue:`324`, :issue:`329`) - -.. _changelog-0.13.2: - -0.13.2 / 2020-05-14 -------------------- - -- Fix ``Provided Schema does not match Table`` error when the existing table - contains required fields. (:issue:`315`) - -.. _changelog-0.13.1: - -0.13.1 / 2020-02-13 -------------------- - -- Fix ``AttributeError`` with BQ Storage API to download empty results. - (:issue:`299`) - -.. _changelog-0.13.0: - -0.13.0 / 2019-12-12 -------------------- - -- Raise ``NotImplementedError`` when the deprecated ``private_key`` argument - is used. (:issue:`301`) - - -.. _changelog-0.12.0: - -0.12.0 / 2019-11-25 -------------------- - -- Add ``max_results`` argument to :func:`~pandas_gbq.read_gbq()`. Use this - argument to limit the number of rows in the results DataFrame. Set - ``max_results`` to 0 to ignore query outputs, such as for DML or DDL - queries. (:issue:`102`) -- Add ``progress_bar_type`` argument to :func:`~pandas_gbq.read_gbq()`. Use - this argument to display a progress bar when downloading data. - (:issue:`182`) - -Dependency updates -~~~~~~~~~~~~~~~~~~ - -- Update the minimum version of ``google-cloud-bigquery`` to 1.11.1. - (:issue:`296`) - -Documentation -~~~~~~~~~~~~~ - -- Add code samples to introduction and refactor howto guides. (:issue:`239`) - - -.. _changelog-0.11.0: - -0.11.0 / 2019-07-29 -------------------- - -- **Breaking Change:** Python 2 support has been dropped. This is to align - with the pandas package which dropped Python 2 support at the end of 2019. - (:issue:`268`) - -Enhancements -~~~~~~~~~~~~ - -- Ensure ``table_schema`` argument is not modified inplace. (:issue:`278`) - -Implementation changes -~~~~~~~~~~~~~~~~~~~~~~ - -- Use object dtype for ``STRING``, ``ARRAY``, and ``STRUCT`` columns when - there are zero rows. (:issue:`285`) - -Internal changes -~~~~~~~~~~~~~~~~ - -- Populate ``user-agent`` with ``pandas`` version information. (:issue:`281`) -- Fix ``pytest.raises`` usage for latest pytest. Fix warnings in tests. - (:issue:`282`) -- Update CI to install nightly packages in the conda tests. (:issue:`254`) - -.. _changelog-0.10.0: - -0.10.0 / 2019-04-05 -------------------- - -- **Breaking Change:** Default SQL dialect is now ``standard``. Use - :attr:`pandas_gbq.context.dialect` to override the default value. - (:issue:`195`, :issue:`245`) - -Documentation -~~~~~~~~~~~~~ - -- Document :ref:`BigQuery data type to pandas dtype conversion - ` for ``read_gbq``. (:issue:`269`) - -Dependency updates -~~~~~~~~~~~~~~~~~~ - -- Update the minimum version of ``google-cloud-bigquery`` to 1.9.0. - (:issue:`247`) -- Update the minimum version of ``pandas`` to 0.19.0. (:issue:`262`) - -Internal changes -~~~~~~~~~~~~~~~~ - -- Update the authentication credentials. **Note:** You may need to set - ``reauth=True`` in order to update your credentials to the most recent - version. This is required to use new functionality such as the BigQuery - Storage API. (:issue:`267`) -- Use ``to_dataframe()`` from ``google-cloud-bigquery`` in the ``read_gbq()`` - function. (:issue:`247`) - -Enhancements -~~~~~~~~~~~~ - -- Fix a bug where pandas-gbq could not upload an empty DataFrame. (:issue:`237`) -- Allow ``table_schema`` in :func:`to_gbq` to contain only a subset of columns, - with the rest being populated using the DataFrame dtypes (:issue:`218`) - (contributed by @johnpaton) -- Read ``project_id`` in :func:`to_gbq` from provided ``credentials`` if - available (contributed by @daureg) -- ``read_gbq`` uses the timezone-aware ``DatetimeTZDtype(unit='ns', - tz='UTC')`` dtype for BigQuery ``TIMESTAMP`` columns. (:issue:`269`) -- Add ``use_bqstorage_api`` to :func:`read_gbq`. The BigQuery Storage API can - be used to download large query results (>125 MB) more quickly. If the BQ - Storage API can't be used, the BigQuery API is used instead. (:issue:`133`, - :issue:`270`) - -.. _changelog-0.9.0: - -0.9.0 / 2019-01-11 ------------------- - -- Warn when deprecated ``private_key`` parameter is used (:issue:`240`) -- **New dependency** Use the ``pydata-google-auth`` package for - authentication. (:issue:`241`) - -.. _changelog-0.8.0: - -0.8.0 / 2018-11-12 ------------------- - -Breaking changes -~~~~~~~~~~~~~~~~ - -- **Deprecate** ``private_key`` parameter to :func:`pandas_gbq.read_gbq` and - :func:`pandas_gbq.to_gbq` in favor of new ``credentials`` argument. Instead, - create a credentials object using - :func:`google.oauth2.service_account.Credentials.from_service_account_info` - or - :func:`google.oauth2.service_account.Credentials.from_service_account_file`. - See the :doc:`authentication how-to guide ` for - examples. (:issue:`161`, :issue:`231`) - -Enhancements -~~~~~~~~~~~~ - -- Allow newlines in data passed to ``to_gbq``. (:issue:`180`) -- Add :attr:`pandas_gbq.context.dialect` to allow overriding the default SQL - syntax dialect. (:issue:`195`, :issue:`235`) -- Support Python 3.7. (:issue:`197`, :issue:`232`) - -Internal changes -~~~~~~~~~~~~~~~~ - -- Migrate tests to CircleCI. (:issue:`228`, :issue:`232`) - -.. _changelog-0.7.0: - -0.7.0 / 2018-10-19 --------------------- - -- `int` columns which contain `NULL` are now cast to `float`, rather than - `object` type. (:issue:`174`) -- `DATE`, `DATETIME` and `TIMESTAMP` columns are now parsed as pandas' `timestamp` - objects (:issue:`224`) -- Add :class:`pandas_gbq.Context` to cache credentials in-memory, across - calls to ``read_gbq`` and ``to_gbq``. (:issue:`198`, :issue:`208`) -- Fast queries now do not log above ``DEBUG`` level. (:issue:`204`) - With BigQuery's release of `clustering `__ - querying smaller samples of data is now faster and cheaper. -- Don't load credentials from disk if reauth is ``True``. (:issue:`212`) - This fixes a bug where pandas-gbq could not refresh credentials if the - cached credentials were invalid, revoked, or expired, even when - ``reauth=True``. -- Catch RefreshError when trying credentials. (:issue:`226`) - -Internal changes -~~~~~~~~~~~~~~~~ - -- Avoid listing datasets and tables in system tests. (:issue:`215`) -- Improved performance from eliminating some duplicative parsing steps - (:issue:`224`) - -.. _changelog-0.6.1: - -0.6.1 / 2018-09-11 --------------------- - -- Improved ``read_gbq`` performance and memory consumption by delegating - ``DataFrame`` construction to the Pandas library, radically reducing - the number of loops that execute in python - (:issue:`128`) -- Reduced verbosity of logging from ``read_gbq``, particularly for short - queries. (:issue:`201`) -- Avoid ``SELECT 1`` query when running ``to_gbq``. (:issue:`202`) - -.. _changelog-0.6.0: - -0.6.0 / 2018-08-15 --------------------- - -- Warn when ``dialect`` is not passed in to ``read_gbq``. The default dialect - will be changing from 'legacy' to 'standard' in a future version. - (:issue:`195`) -- Use general float with 15 decimal digit precision when writing to local - CSV buffer in ``to_gbq``. This prevents numerical overflow in certain - edge cases. (:issue:`192`) - -.. _changelog-0.5.0: - -0.5.0 / 2018-06-15 ------------------- - -- Project ID parameter is optional in ``read_gbq`` and ``to_gbq`` when it can - inferred from the environment. Note: you must still pass in a project ID when - using user-based authentication. (:issue:`103`) -- Progress bar added for ``to_gbq``, through an optional library `tqdm` as - dependency. (:issue:`162`) -- Add location parameter to ``read_gbq`` and ``to_gbq`` so that pandas-gbq - can work with datasets in the Tokyo region. (:issue:`177`) - -Documentation -~~~~~~~~~~~~~ - -- Add :doc:`authentication how-to guide `. (:issue:`183`) -- Update :doc:`contributing` guide with new paths to tests. (:issue:`154`, - :issue:`164`) - -Internal changes -~~~~~~~~~~~~~~~~ - -- Tests now use `nox` to run in multiple Python environments. (:issue:`52`) -- Renamed internal modules. (:issue:`154`) -- Refactored auth to an internal auth module. (:issue:`176`) -- Add unit tests for ``get_credentials()``. (:issue:`184`) - -.. _changelog-0.4.1: - -0.4.1 / 2018-04-05 ------------------- - -- Only show ``verbose`` deprecation warning if Pandas version does not - populate it. (:issue:`157`) - -.. _changelog-0.4.0: - -0.4.0 / 2018-04-03 ------------------- - -- Fix bug in `read_gbq` when building a dataframe with integer columns - on Windows. Explicitly use 64bit integers when converting from BQ types. - (:issue:`119`) -- Fix bug in `read_gbq` when querying for an array of floats (:issue:`123`) -- Fix bug in `read_gbq` with configuration argument. Updates `read_gbq` to - account for breaking change in the way ``google-cloud-python`` version - 0.32.0+ handles query configuration API representation. (:issue:`152`) -- Fix bug in `to_gbq` where seconds were discarded in timestamp columns. - (:issue:`148`) -- Fix bug in `to_gbq` when supplying a user-defined schema (:issue:`150`) -- **Deprecate** the ``verbose`` parameter in `read_gbq` and `to_gbq`. - Messages use the logging module instead of printing progress directly to - standard output. (:issue:`12`) - -.. _changelog-0.3.1: - -0.3.1 / 2018-02-13 ------------------- - -- Fix an issue where Unicode couldn't be uploaded in Python 2 (:issue:`106`) -- Add support for a passed schema in :func:``to_gbq`` instead inferring the schema from the passed ``DataFrame`` with ``DataFrame.dtypes`` (:issue:`46`) -- Fix an issue where a dataframe containing both integer and floating point columns could not be uploaded with ``to_gbq`` (:issue:`116`) -- ``to_gbq`` now uses ``to_csv`` to avoid manually looping over rows in a dataframe (should result in faster table uploads) (:issue:`96`) - -.. _changelog-0.3.0: - -0.3.0 / 2018-01-03 ------------------- - -- Use the `google-cloud-bigquery `__ library for API calls. The ``google-cloud-bigquery`` package is a new dependency, and dependencies on ``google-api-python-client`` and ``httplib2`` are removed. See the `installation guide `__ for more details. (:issue:`93`) -- Structs and arrays are now named properly (:issue:`23`) and BigQuery functions like ``array_agg`` no longer run into errors during type conversion (:issue:`22`). -- :func:`to_gbq` now uses a load job instead of the streaming API. Remove ``StreamingInsertError`` class, as it is no longer used by :func:`to_gbq`. (:issue:`7`, :issue:`75`) - -.. _changelog-0.2.1: - -0.2.1 / 2017-11-27 ------------------- - -- :func:`read_gbq` now raises ``QueryTimeout`` if the request exceeds the ``query.timeoutMs`` value specified in the BigQuery configuration. (:issue:`76`) -- Environment variable ``PANDAS_GBQ_CREDENTIALS_FILE`` can now be used to override the default location where the BigQuery user account credentials are stored. (:issue:`86`) -- BigQuery user account credentials are now stored in an application-specific hidden user folder on the operating system. (:issue:`41`) - -.. _changelog-0.2.0: - -0.2.0 / 2017-07-24 ------------------- - -- Drop support for Python 3.4 (:issue:`40`) -- The dataframe passed to ```.to_gbq(...., if_exists='append')``` needs to contain only a subset of the fields in the BigQuery schema. (:issue:`24`) -- Use the `google-auth `__ library for authentication because ``oauth2client`` is deprecated. (:issue:`39`) -- :func:`read_gbq` now has a ``auth_local_webserver`` boolean argument for controlling whether to use web server or console flow when getting user credentials. Replaces `--noauth_local_webserver` command line argument. (:issue:`35`) -- :func:`read_gbq` now displays the BigQuery Job ID and standard price in verbose output. (:issue:`70` and :issue:`71`) - -.. _changelog-0.1.6: - -0.1.6 / 2017-05-03 ------------------- - -- All gbq errors will simply be subclasses of ``ValueError`` and no longer inherit from the deprecated ``PandasError``. - -.. _changelog-0.1.4: - -0.1.4 / 2017-03-17 ------------------- - -- ``InvalidIndexColumn`` will be raised instead of ``InvalidColumnOrder`` in :func:`read_gbq` when the index column specified does not exist in the BigQuery schema. (:issue:`6`) - -.. _changelog-0.1.3: - -0.1.3 / 2017-03-04 ------------------- - -- Bug with appending to a BigQuery table where fields have modes (NULLABLE,REQUIRED,REPEATED) specified. These modes were compared versus the remote schema and writing a table via :func:`to_gbq` would previously raise. (:issue:`13`) - -.. _changelog-0.1.2: - -0.1.2 / 2017-02-23 ------------------- - -Initial release of transfered code from `pandas `__ - -Includes patches since the 0.19.2 release on pandas with the following: - -- :func:`read_gbq` now allows query configuration preferences `pandas-GH#14742 `__ -- :func:`read_gbq` now stores ``INTEGER`` columns as ``dtype=object`` if they contain ``NULL`` values. Otherwise they are stored as ``int64``. This prevents precision lost for integers greather than 2**53. Furthermore ``FLOAT`` columns with values above 10**4 are no longer casted to ``int64`` which also caused precision loss `pandas-GH#14064 `__, and `pandas-GH#14305 `__ diff --git a/packages/pandas-gbq/docs/source/conf.py b/packages/pandas-gbq/docs/source/conf.py index afad588db751..bfcc94efe87d 100644 --- a/packages/pandas-gbq/docs/source/conf.py +++ b/packages/pandas-gbq/docs/source/conf.py @@ -49,6 +49,7 @@ "sphinx.ext.intersphinx", "sphinx.ext.coverage", "sphinx.ext.ifconfig", + "recommonmark", ] # Add any paths that contain templates here, relative to this directory. @@ -58,7 +59,7 @@ # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = ".rst" +source_suffix = [".rst", ".md"] # The encoding of source files. # diff --git a/packages/pandas-gbq/docs/source/contributing.rst b/packages/pandas-gbq/docs/source/contributing.rst index cacbf1c4d6c5..3bd868495940 100644 --- a/packages/pandas-gbq/docs/source/contributing.rst +++ b/packages/pandas-gbq/docs/source/contributing.rst @@ -334,10 +334,8 @@ run gbq integration tests on a forked repository: Documenting your code --------------------- -Changes should be reflected in the release notes located in ``doc/source/changelog.rst``. -This file contains an ongoing change log. Add an entry to this file to document your fix, -enhancement or (unavoidable) breaking change. Make sure to include the GitHub issue number -when adding your entry (using `` :issue:`1234` `` where `1234` is the issue/pull request number). +Changes should follow convential commits. The release-please bot uses the +commit message to create an ongoing change log. If your code is an enhancement, it is most likely necessary to add usage examples to the existing documentation. Further, to let users know when diff --git a/packages/pandas-gbq/docs/source/index.rst b/packages/pandas-gbq/docs/source/index.rst index bfb51d9eadda..e104127d9cd4 100644 --- a/packages/pandas-gbq/docs/source/index.rst +++ b/packages/pandas-gbq/docs/source/index.rst @@ -39,7 +39,7 @@ Contents: writing.rst api.rst contributing.rst - changelog.rst + changelog.md privacy.rst diff --git a/packages/pandas-gbq/release-procedure.md b/packages/pandas-gbq/release-procedure.md index b682db2fa3c6..3b33021d5d07 100644 --- a/packages/pandas-gbq/release-procedure.md +++ b/packages/pandas-gbq/release-procedure.md @@ -1,8 +1,6 @@ * Send PR to prepare release on scheduled date. - * Add current date and any missing changes to [`docs/source/changelog.rst`](https://github.com/pydata/pandas-gbq/blob/master/docs/source/changelog.rst). - * Verify your local repository is on the latest changes. `rebase -i` should be noop. git fetch pandas-gbq master @@ -37,7 +35,6 @@ * Create the [release on GitHub](https://github.com/pydata/pandas-gbq/releases/new) using the tag created earlier. - * Copy release notes from [changelog.rst](https://github.com/pydata/pandas-gbq/blob/master/docs/source/changelog.rst). * Upload wheel and source zip from `dist/` directory. * Do a pull-request to the feedstock on `pandas-gbq-feedstock `__ From 0fb0a2e859fa45941a35611f1cc59fa0a15fb154 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 26 Aug 2021 15:54:07 -0500 Subject: [PATCH 229/519] refactor: configure `setup.py` in preparation for using release-please (#371) * refactor: configure release-please * blacken --- .../pandas-gbq/.github/release-please.yml | 1 + packages/pandas-gbq/pandas_gbq/__init__.py | 15 +- packages/pandas-gbq/pandas_gbq/auth.py | 4 +- packages/pandas-gbq/pandas_gbq/exceptions.py | 1 + packages/pandas-gbq/pandas_gbq/version.py | 5 + packages/pandas-gbq/setup.cfg | 39 +- packages/pandas-gbq/setup.py | 83 +- packages/pandas-gbq/versioneer.py | 1909 ----------------- 8 files changed, 90 insertions(+), 1967 deletions(-) create mode 100644 packages/pandas-gbq/.github/release-please.yml create mode 100644 packages/pandas-gbq/pandas_gbq/version.py delete mode 100644 packages/pandas-gbq/versioneer.py diff --git a/packages/pandas-gbq/.github/release-please.yml b/packages/pandas-gbq/.github/release-please.yml new file mode 100644 index 000000000000..4507ad0598a5 --- /dev/null +++ b/packages/pandas-gbq/.github/release-please.yml @@ -0,0 +1 @@ +releaseType: python diff --git a/packages/pandas-gbq/pandas_gbq/__init__.py b/packages/pandas-gbq/pandas_gbq/__init__.py index 33f08cf42ba9..df2b603d6496 100644 --- a/packages/pandas-gbq/pandas_gbq/__init__.py +++ b/packages/pandas-gbq/pandas_gbq/__init__.py @@ -4,9 +4,14 @@ from .gbq import to_gbq, read_gbq, Context, context # noqa -from ._version import get_versions +from pandas_gbq import version as pandas_gbq_version -versions = get_versions() -__version__ = versions.get("closest-tag", versions["version"]) -__git_revision__ = versions["full-revisionid"] -del get_versions, versions +__version__ = pandas_gbq_version.__version__ + +__all__ = [ + "__version__", + "to_gbq", + "read_gbq", + "Context", + "context", +] diff --git a/packages/pandas-gbq/pandas_gbq/auth.py b/packages/pandas-gbq/pandas_gbq/auth.py index 5215378737cd..61dcee06ffa4 100644 --- a/packages/pandas-gbq/pandas_gbq/auth.py +++ b/packages/pandas-gbq/pandas_gbq/auth.py @@ -55,9 +55,7 @@ def get_credentials( return credentials, project_id -def get_credentials_cache( - reauth, -): +def get_credentials_cache(reauth): import pydata_google_auth.cache if reauth: diff --git a/packages/pandas-gbq/pandas_gbq/exceptions.py b/packages/pandas-gbq/pandas_gbq/exceptions.py index aa83cdd3c34d..aec0ea1a2577 100644 --- a/packages/pandas-gbq/pandas_gbq/exceptions.py +++ b/packages/pandas-gbq/pandas_gbq/exceptions.py @@ -2,6 +2,7 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. + class AccessDenied(ValueError): """ Raised when invalid credentials are provided, or tokens have expired. diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py new file mode 100644 index 000000000000..4094268e403f --- /dev/null +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -0,0 +1,5 @@ +# Copyright (c) 2021 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +__version__ = "0.15.0" diff --git a/packages/pandas-gbq/setup.cfg b/packages/pandas-gbq/setup.cfg index 1f1185b3d6fb..c3a2b39f6528 100644 --- a/packages/pandas-gbq/setup.cfg +++ b/packages/pandas-gbq/setup.cfg @@ -1,22 +1,19 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -# See the docstring in versioneer.py for instructions. Note that you must -# re-run 'versioneer.py setup' after changing this section, and commit the -# resulting files. - -[versioneer] -VCS = git -style = pep440 -versionfile_source = pandas_gbq/_version.py -versionfile_build = pandas_gbq/_version.py -tag_prefix = -parentdir_prefix = pandas_gbq- - -[flake8] -ignore = E731, W503 -exclude = docs - -[isort] -multi_line_output=3 -line_length=79 -default_section=THIRDPARTY -known_first_party=pandas_gbq +# Generated by synthtool. DO NOT EDIT! +[bdist_wheel] +universal = 1 diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index 224dc0076642..a7e23eece824 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -5,22 +5,23 @@ # -*- coding: utf-8 -*- -import versioneer -from setuptools import find_packages, setup +import io +import os -NAME = "pandas-gbq" +import setuptools -# versioning -cmdclass = versioneer.get_cmdclass() +# Package metadata. +name = "pandas-gbq" +description = "Google BigQuery connector for pandas" -def readme(): - with open("README.rst") as f: - return f.read() - - -INSTALL_REQUIRES = [ +# Should be one of: +# 'Development Status :: 3 - Alpha' +# 'Development Status :: 4 - Beta' +# 'Development Status :: 5 - Production/Stable' +release_status = "Development Status :: 4 - Beta" +dependencies = [ "setuptools", "pandas>=0.23.2", "pydata-google-auth", @@ -30,35 +31,59 @@ def readme(): # https://github.com/pydata/pandas-gbq/issues/343 "google-cloud-bigquery[bqstorage,pandas]>=1.11.1,<3.0.0dev,!=2.4.*", ] - extras = {"tqdm": "tqdm>=4.23.0"} -setup( - name=NAME, - version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), - description="Pandas interface to Google BigQuery", - long_description=readme(), - license="BSD License", - author="The PyData Development Team", - author_email="pydata@googlegroups.com", - url="https://github.com/pydata/pandas-gbq", +# Setup boilerplate below this line. + +package_root = os.path.abspath(os.path.dirname(__file__)) + +readme_filename = os.path.join(package_root, "README.rst") +with io.open(readme_filename, encoding="utf-8") as readme_file: + readme = readme_file.read() + +version = {} +with open(os.path.join(package_root, "pandas_gbq/version.py")) as fp: + exec(fp.read(), version) +version = version["__version__"] + +# Only include packages under the 'google' namespace. Do not include tests, +# benchmarks, etc. +packages = [ + package + for package in setuptools.PEP420PackageFinder.find() + if package.startswith("pandas_gbq") +] + + +setuptools.setup( + name=name, + version=version, + description=description, + long_description=readme, + author="pandas-gbq authors", + author_email="googleapis-packages@google.com", + license="BSD-3-Clause", + url="https://github.com/googleapis/python-bigquery-pandas", classifiers=[ - "Development Status :: 4 - Beta", - "Environment :: Console", + release_status, + "Intended Audience :: Developers", "Intended Audience :: Science/Research", "Operating System :: OS Independent", + "License :: OSI Approved :: BSD License", "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Operating System :: OS Independent", + "Topic :: Internet", "Topic :: Scientific/Engineering", ], - keywords="data", - install_requires=INSTALL_REQUIRES, + platforms="Posix; MacOS X; Windows", + packages=packages, + install_requires=dependencies, extras_require=extras, - python_requires=">=3.7", - packages=find_packages(exclude=["contrib", "docs", "tests*"]), - test_suite="tests", + python_requires=">=3.7, <3.10", + include_package_data=True, + zip_safe=False, ) diff --git a/packages/pandas-gbq/versioneer.py b/packages/pandas-gbq/versioneer.py deleted file mode 100644 index 2968e97bc824..000000000000 --- a/packages/pandas-gbq/versioneer.py +++ /dev/null @@ -1,1909 +0,0 @@ -# Copyright (c) 2017 pandas-gbq Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - - -# Version: 0.18 - -"""The Versioneer - like a rocketeer, but for versions. - -The Versioneer -============== - -* like a rocketeer, but for versions! -* https://github.com/warner/python-versioneer -* Brian Warner -* License: Public Domain -* Compatible With: python2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, and pypy -* [![Latest Version] -(https://pypip.in/version/versioneer/badge.svg?style=flat) -](https://pypi.python.org/pypi/versioneer/) -* [![Build Status] -(https://travis-ci.org/warner/python-versioneer.png?branch=master) -](https://travis-ci.org/warner/python-versioneer) - -This is a tool for managing a recorded version number in distutils-based -python projects. The goal is to remove the tedious and error-prone "update -the embedded version string" step from your release process. Making a new -release should be as easy as recording a new tag in your version-control -system, and maybe making new tarballs. - - -## Quick Install - -* `pip install versioneer` to somewhere to your $PATH -* add a `[versioneer]` section to your setup.cfg (see below) -* run `versioneer install` in your source tree, commit the results - -## Version Identifiers - -Source trees come from a variety of places: - -* a version-control system checkout (mostly used by developers) -* a nightly tarball, produced by build automation -* a snapshot tarball, produced by a web-based VCS browser, like github's - "tarball from tag" feature -* a release tarball, produced by "setup.py sdist", distributed through PyPI - -Within each source tree, the version identifier (either a string or a number, -this tool is format-agnostic) can come from a variety of places: - -* ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows - about recent "tags" and an absolute revision-id -* the name of the directory into which the tarball was unpacked -* an expanded VCS keyword ($Id$, etc) -* a `_version.py` created by some earlier build step - -For released software, the version identifier is closely related to a VCS -tag. Some projects use tag names that include more than just the version -string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool -needs to strip the tag prefix to extract the version identifier. For -unreleased software (between tags), the version identifier should provide -enough information to help developers recreate the same tree, while also -giving them an idea of roughly how old the tree is (after version 1.2, before -version 1.3). Many VCS systems can report a description that captures this, -for example `git describe --tags --dirty --always` reports things like -"0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the -0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has -uncommitted changes. - -The version identifier is used for multiple purposes: - -* to allow the module to self-identify its version: `myproject.__version__` -* to choose a name and prefix for a 'setup.py sdist' tarball - -## Theory of Operation - -Versioneer works by adding a special `_version.py` file into your source -tree, where your `__init__.py` can import it. This `_version.py` knows how to -dynamically ask the VCS tool for version information at import time. - -`_version.py` also contains `$Revision$` markers, and the installation -process marks `_version.py` to have this marker rewritten with a tag name -during the `git archive` command. As a result, generated tarballs will -contain enough information to get the proper version. - -To allow `setup.py` to compute a version too, a `versioneer.py` is added to -the top level of your source tree, next to `setup.py` and the `setup.cfg` -that configures it. This overrides several distutils/setuptools commands to -compute the version when invoked, and changes `setup.py build` and `setup.py -sdist` to replace `_version.py` with a small static file that contains just -the generated version data. - -## Installation - -See [INSTALL.md](./INSTALL.md) for detailed installation instructions. - -## Version-String Flavors - -Code which uses Versioneer can learn about its version string at runtime by -importing `_version` from your main `__init__.py` file and running the -`get_versions()` function. From the "outside" (e.g. in `setup.py`), you can -import the top-level `versioneer.py` and run `get_versions()`. - -Both functions return a dictionary with different flavors of version -information: - -* `['version']`: A condensed version string, rendered using the selected - style. This is the most commonly used value for the project's version - string. The default "pep440" style yields strings like `0.11`, - `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section - below for alternative styles. - -* `['full-revisionid']`: detailed revision identifier. For Git, this is the - full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac". - -* `['date']`: Date and time of the latest `HEAD` commit. For Git, it is the - commit date in ISO 8601 format. This will be None if the date is not - available. - -* `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that - this is only accurate if run in a VCS checkout, otherwise it is likely to - be False or None - -* `['error']`: if the version string could not be computed, this will be set - to a string describing the problem, otherwise it will be None. It may be - useful to throw an exception in setup.py if this is set, to avoid e.g. - creating tarballs with a version string of "unknown". - -Some variants are more useful than others. Including `full-revisionid` in a -bug report should allow developers to reconstruct the exact code being tested -(or indicate the presence of local changes that should be shared with the -developers). `version` is suitable for display in an "about" box or a CLI -`--version` output: it can be easily compared against release notes and lists -of bugs fixed in various releases. - -The installer adds the following text to your `__init__.py` to place a basic -version in `YOURPROJECT.__version__`: - - from ._version import get_versions - __version__ = get_versions()['version'] - del get_versions - -## Styles - -The setup.cfg `style=` configuration controls how the VCS information is -rendered into a version string. - -The default style, "pep440", produces a PEP440-compliant string, equal to the -un-prefixed tag name for actual releases, and containing an additional "local -version" section with more detail for in-between builds. For Git, this is -TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags ---dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the -tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and -that this commit is two revisions ("+2") beyond the "0.11" tag. For released -software (exactly equal to a known tag), the identifier will only contain the -stripped tag, e.g. "0.11". - -Other styles are available. See [details.md](details.md) in the Versioneer -source tree for descriptions. - -## Debugging - -Versioneer tries to avoid fatal errors: if something goes wrong, it will tend -to return a version of "0+unknown". To investigate the problem, run `setup.py -version`, which will run the version-lookup code in a verbose mode, and will -display the full contents of `get_versions()` (including the `error` string, -which may help identify what went wrong). - -## Known Limitations - -Some situations are known to cause problems for Versioneer. This details the -most significant ones. More can be found on Github -[issues page](https://github.com/warner/python-versioneer/issues). - -### Subprojects - -Versioneer has limited support for source trees in which `setup.py` is not in -the root directory (e.g. `setup.py` and `.git/` are *not* siblings). The are -two common reasons why `setup.py` might not be in the root: - -* Source trees which contain multiple subprojects, such as - [Buildbot](https://github.com/buildbot/buildbot), which contains both - "master" and "slave" subprojects, each with their own `setup.py`, - `setup.cfg`, and `tox.ini`. Projects like these produce multiple PyPI - distributions (and upload multiple independently-installable tarballs). -* Source trees whose main purpose is to contain a C library, but which also - provide bindings to Python (and perhaps other langauges) in subdirectories. - -Versioneer will look for `.git` in parent directories, and most operations -should get the right version string. However `pip` and `setuptools` have bugs -and implementation details which frequently cause `pip install .` from a -subproject directory to fail to find a correct version string (so it usually -defaults to `0+unknown`). - -`pip install --editable .` should work correctly. `setup.py install` might -work too. - -Pip-8.1.1 is known to have this problem, but hopefully it will get fixed in -some later version. - -[Bug #38](https://github.com/warner/python-versioneer/issues/38) is tracking -this issue. The discussion in -[PR #61](https://github.com/warner/python-versioneer/pull/61) describes the -issue from the Versioneer side in more detail. -[pip PR#3176](https://github.com/pypa/pip/pull/3176) and -[pip PR#3615](https://github.com/pypa/pip/pull/3615) contain work to improve -pip to let Versioneer work correctly. - -Versioneer-0.16 and earlier only looked for a `.git` directory next to the -`setup.cfg`, so subprojects were completely unsupported with those releases. - -### Editable installs with setuptools <= 18.5 - -`setup.py develop` and `pip install --editable .` allow you to install a -project into a virtualenv once, then continue editing the source code (and -test) without re-installing after every change. - -"Entry-point scripts" (`setup(entry_points={"console_scripts": ..})`) are a -convenient way to specify executable scripts that should be installed along -with the python package. - -These both work as expected when using modern setuptools. When using -setuptools-18.5 or earlier, however, certain operations will cause -`pkg_resources.DistributionNotFound` errors when running the entrypoint -script, which must be resolved by re-installing the package. This happens -when the install happens with one version, then the egg_info data is -regenerated while a different version is checked out. Many setup.py commands -cause egg_info to be rebuilt (including `sdist`, `wheel`, and installing into -a different virtualenv), so this can be surprising. - -[Bug #83](https://github.com/warner/python-versioneer/issues/83) describes -this one, but upgrading to a newer version of setuptools should probably -resolve it. - -### Unicode version strings - -While Versioneer works (and is continually tested) with both Python 2 and -Python 3, it is not entirely consistent with bytes-vs-unicode distinctions. -Newer releases probably generate unicode version strings on py2. It's not -clear that this is wrong, but it may be surprising for applications when then -write these strings to a network connection or include them in bytes-oriented -APIs like cryptographic checksums. - -[Bug #71](https://github.com/warner/python-versioneer/issues/71) investigates -this question. - - -## Updating Versioneer - -To upgrade your project to a new release of Versioneer, do the following: - -* install the new Versioneer (`pip install -U versioneer` or equivalent) -* edit `setup.cfg`, if necessary, to include any new configuration settings - indicated by the release notes. See [UPGRADING](./UPGRADING.md) for details. -* re-run `versioneer install` in your source tree, to replace - `SRC/_version.py` -* commit any changed files - -## Future Directions - -This tool is designed to make it easily extended to other version-control -systems: all VCS-specific components are in separate directories like -src/git/ . The top-level `versioneer.py` script is assembled from these -components by running make-versioneer.py . In the future, make-versioneer.py -will take a VCS name as an argument, and will construct a version of -`versioneer.py` that is specific to the given VCS. It might also take the -configuration arguments that are currently provided manually during -installation by editing setup.py . Alternatively, it might go the other -direction and include code from all supported VCS systems, reducing the -number of intermediate scripts. - - -## License - -To make Versioneer easier to embed, all its code is dedicated to the public -domain. The `_version.py` that it creates is also in the public domain. -Specifically, both are released under the Creative Commons "Public Domain -Dedication" license (CC0-1.0), as described in -https://creativecommons.org/publicdomain/zero/1.0/ . - -""" - -from __future__ import print_function - -import errno -import json -import os -import re -import subprocess -import sys - -try: - import configparser -except ImportError: - import ConfigParser as configparser - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_root(): - """Get the project root directory. - - We require that all commands are run from the project root, i.e. the - directory that contains setup.py, setup.cfg, and versioneer.py . - """ - root = os.path.realpath(os.path.abspath(os.getcwd())) - setup_py = os.path.join(root, "setup.py") - versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - # allow 'python path/to/setup.py COMMAND' - root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) - setup_py = os.path.join(root, "setup.py") - versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - err = ( - "Versioneer was unable to run the project root directory. " - "Versioneer requires setup.py to be executed from " - "its immediate directory (like 'python setup.py COMMAND'), " - "or in a way that lets it use sys.argv[0] to find the root " - "(like 'python path/to/setup.py COMMAND')." - ) - raise VersioneerBadRootError(err) - try: - # Certain runtime workflows (setup.py install/develop in a setuptools - # tree) execute all dependencies in a single python process, so - # "versioneer" may be imported multiple times, and python's shared - # module-import table will cache the first one. So we can't use - # os.path.dirname(__file__), as that will find whichever - # versioneer.py was first imported, even in later projects. - me = os.path.realpath(os.path.abspath(__file__)) - me_dir = os.path.normcase(os.path.splitext(me)[0]) - vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) - if me_dir != vsr_dir: - print( - "Warning: build in %s is using versioneer.py from %s" - % (os.path.dirname(me), versioneer_py) - ) - except NameError: - pass - return root - - -def get_config_from_root(root): - """Read the project setup.cfg file to determine Versioneer config.""" - # This might raise EnvironmentError (if setup.cfg is missing), or - # configparser.NoSectionError (if it lacks a [versioneer] section), or - # configparser.NoOptionError (if it lacks "VCS="). See the docstring at - # the top of versioneer.py for instructions on writing your setup.cfg . - setup_cfg = os.path.join(root, "setup.cfg") - parser = configparser.SafeConfigParser() - with open(setup_cfg, "r") as f: - parser.readfp(f) - VCS = parser.get("versioneer", "VCS") # mandatory - - def get(parser, name): - if parser.has_option("versioneer", name): - return parser.get("versioneer", name) - return None - - cfg = VersioneerConfig() - cfg.VCS = VCS - cfg.style = get(parser, "style") or "" - cfg.versionfile_source = get(parser, "versionfile_source") - cfg.versionfile_build = get(parser, "versionfile_build") - cfg.tag_prefix = get(parser, "tag_prefix") - if cfg.tag_prefix in ("''", '""'): - cfg.tag_prefix = "" - cfg.parentdir_prefix = get(parser, "parentdir_prefix") - cfg.verbose = get(parser, "verbose") - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -# these dictionaries contain VCS-specific tools -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - - return decorate - - -def run_command( - commands, args, cwd=None, verbose=False, hide_stderr=False, env=None -): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen( - [c] + args, - cwd=cwd, - env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr else None), - ) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %s" % (commands,)) - return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %s (error)" % dispcmd) - print("stdout was %s" % stdout) - return None, p.returncode - return stdout, p.returncode - - -LONG_VERSION_PY[ - "git" -] = ''' -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (built by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. - -# This file is released into the public domain. Generated by -# versioneer-0.18 (https://github.com/warner/python-versioneer) - -"""Git implementation of _version.py.""" - -import errno -import os -import re -import subprocess -import sys - - -def get_keywords(): - """Get the keywords needed to look up the version information.""" - # these strings will be replaced by git during git-archive. - # setup.py/versioneer.py will grep for the variable names, so they must - # each be defined on a line of their own. _version.py will just call - # get_keywords(). - git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" - git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" - git_date = "%(DOLLAR)sFormat:%%ci%(DOLLAR)s" - keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} - return keywords - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_config(): - """Create, populate and return the VersioneerConfig() object.""" - # these strings are filled in when 'setup.py versioneer' creates - # _version.py - cfg = VersioneerConfig() - cfg.VCS = "git" - cfg.style = "%(STYLE)s" - cfg.tag_prefix = "%(TAG_PREFIX)s" - cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s" - cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s" - cfg.verbose = False - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - return decorate - - -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %%s" %% dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %%s" %% (commands,)) - return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %%s (error)" %% dispcmd) - print("stdout was %%s" %% stdout) - return None, p.returncode - return stdout, p.returncode - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for i in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print("Tried directories %%s but none started with prefix %%s" %% - (str(rootdirs), parentdir_prefix)) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - date = keywords.get("date") - if date is not None: - # git-2.2.0 added "%%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %%d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) - if verbose: - print("discarding '%%s', no digits" %% ",".join(refs - tags)) - if verbose: - print("likely tags: %%s" %% ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] - if verbose: - print("picking %%s" %% r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) - if rc != 0: - if verbose: - print("Directory %%s not under git control" %% root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%%s*" %% tag_prefix], - cwd=root) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%%s'" - %% describe_out) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%%s' doesn't start with prefix '%%s'" - print(fmt %% (full_tag, tag_prefix)) - pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'" - %% (full_tag, tag_prefix)) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) - pieces["distance"] = int(count_out) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%%ci", "HEAD"], - cwd=root)[0].strip() - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%%d" %% pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%%d" %% pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%%s" %% pieces["short"] - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%%s" %% pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Eexceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%%s'" %% style) - - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} - - -def get_versions(): - """Get version information or return default if unable to do so.""" - # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have - # __file__, we can work backwards from there to the root. Some - # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded keywords. - - cfg = get_config() - verbose = cfg.verbose - - try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, - verbose) - except NotThisMethod: - pass - - try: - root = os.path.realpath(__file__) - # versionfile_source is the relative path from the top of the source - # tree (where the .git directory might live) to this file. Invert - # this to find the root from __file__. - for i in cfg.versionfile_source.split('/'): - root = os.path.dirname(root) - except NameError: - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None} - - try: - pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) - return render(pieces, cfg.style) - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - except NotThisMethod: - pass - - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", "date": None} -''' - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - date = keywords.get("date") - if date is not None: - # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r"\d", r)]) - if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix) :] - if verbose: - print("picking %s" % r) - return { - "version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": None, - "date": date, - } - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return { - "version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": "no suitable tags", - "date": None, - } - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - out, rc = run_command( - GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True - ) - if rc != 0: - if verbose: - print("Directory %s not under git control" % root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command( - GITS, - [ - "describe", - "--tags", - "--dirty", - "--always", - "--long", - "--match", - "%s*" % tag_prefix, - ], - cwd=root, - ) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[: git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ( - "unable to parse git-describe output: '%s'" % describe_out - ) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%s' doesn't start with prefix '%s'" - print(fmt % (full_tag, tag_prefix)) - pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( - full_tag, - tag_prefix, - ) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix) :] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out, rc = run_command( - GITS, ["rev-list", "HEAD", "--count"], cwd=root - ) - pieces["distance"] = int(count_out) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ - 0 - ].strip() - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def do_vcs_install(manifest_in, versionfile_source, ipy): - """Git-specific installation logic for Versioneer. - - For Git, this means creating/changing .gitattributes to mark _version.py - for export-subst keyword substitution. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - files = [manifest_in, versionfile_source] - if ipy: - files.append(ipy) - try: - me = __file__ - if me.endswith(".pyc") or me.endswith(".pyo"): - me = os.path.splitext(me)[0] + ".py" - versioneer_file = os.path.relpath(me) - except NameError: - versioneer_file = "versioneer.py" - files.append(versioneer_file) - present = False - try: - f = open(".gitattributes", "r") - for line in f.readlines(): - if line.strip().startswith(versionfile_source): - if "export-subst" in line.strip().split()[1:]: - present = True - f.close() - except EnvironmentError: - pass - if not present: - f = open(".gitattributes", "a+") - f.write("%s export-subst\n" % versionfile_source) - f.close() - files.append(".gitattributes") - run_command(GITS, ["add", "--"] + files) - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for i in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return { - "version": dirname[len(parentdir_prefix) :], - "full-revisionid": None, - "dirty": False, - "error": None, - "date": None, - } - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print( - "Tried directories %s but none started with prefix %s" - % (str(rootdirs), parentdir_prefix) - ) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -SHORT_VERSION_PY = """ -# This file was generated by 'versioneer.py' (0.18) from -# revision-control system data, or from the parent directory name of an -# unpacked source archive. Distribution tarballs contain a pre-generated copy -# of this file. - -import json - -version_json = ''' -%s -''' # END VERSION_JSON - - -def get_versions(): - return json.loads(version_json) -""" - - -def versions_from_file(filename): - """Try to determine the version from _version.py if present.""" - try: - with open(filename) as f: - contents = f.read() - except EnvironmentError: - raise NotThisMethod("unable to read _version.py") - mo = re.search( - r"version_json = '''\n(.*)''' # END VERSION_JSON", - contents, - re.M | re.S, - ) - if not mo: - mo = re.search( - r"version_json = '''\r\n(.*)''' # END VERSION_JSON", - contents, - re.M | re.S, - ) - if not mo: - raise NotThisMethod("no version_json in _version.py") - return json.loads(mo.group(1)) - - -def write_to_version_file(filename, versions): - """Write the given version number to the given _version.py file.""" - os.unlink(filename) - contents = json.dumps( - versions, sort_keys=True, indent=1, separators=(",", ": ") - ) - with open(filename, "w") as f: - f.write(SHORT_VERSION_PY % contents) - - print("set %s to '%s'" % (filename, versions["version"])) - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%d" % pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%d" % pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Eexceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return { - "version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None, - } - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%s'" % style) - - return { - "version": rendered, - "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], - "error": None, - "date": pieces.get("date"), - } - - -class VersioneerBadRootError(Exception): - """The project root directory is unknown or missing key files.""" - - -def get_versions(verbose=False): - """Get the project version from whatever source is available. - - Returns dict with two keys: 'version' and 'full'. - """ - if "versioneer" in sys.modules: - # see the discussion in cmdclass.py:get_cmdclass() - del sys.modules["versioneer"] - - root = get_root() - cfg = get_config_from_root(root) - - assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg" - handlers = HANDLERS.get(cfg.VCS) - assert handlers, "unrecognized VCS '%s'" % cfg.VCS - verbose = verbose or cfg.verbose - assert ( - cfg.versionfile_source is not None - ), "please set versioneer.versionfile_source" - assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" - - versionfile_abs = os.path.join(root, cfg.versionfile_source) - - # extract version from first of: _version.py, VCS command (e.g. 'git - # describe'), parentdir. This is meant to work for developers using a - # source checkout, for users of a tarball created by 'setup.py sdist', - # and for users of a tarball/zipball created by 'git archive' or github's - # download-from-tag feature or the equivalent in other VCSes. - - get_keywords_f = handlers.get("get_keywords") - from_keywords_f = handlers.get("keywords") - if get_keywords_f and from_keywords_f: - try: - keywords = get_keywords_f(versionfile_abs) - ver = from_keywords_f(keywords, cfg.tag_prefix, verbose) - if verbose: - print("got version from expanded keyword %s" % ver) - return ver - except NotThisMethod: - pass - - try: - ver = versions_from_file(versionfile_abs) - if verbose: - print("got version from file %s %s" % (versionfile_abs, ver)) - return ver - except NotThisMethod: - pass - - from_vcs_f = handlers.get("pieces_from_vcs") - if from_vcs_f: - try: - pieces = from_vcs_f(cfg.tag_prefix, root, verbose) - ver = render(pieces, cfg.style) - if verbose: - print("got version from VCS %s" % ver) - return ver - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - if verbose: - print("got version from parentdir %s" % ver) - return ver - except NotThisMethod: - pass - - if verbose: - print("unable to compute version") - - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", - "date": None, - } - - -def get_version(): - """Get the short version string for this project.""" - return get_versions()["version"] - - -def get_cmdclass(): - """Get the custom setuptools/distutils subclasses used by Versioneer.""" - if "versioneer" in sys.modules: - del sys.modules["versioneer"] - # this fixes the "python setup.py develop" case (also 'install' and - # 'easy_install .'), in which subdependencies of the main project are - # built (using setup.py bdist_egg) in the same python process. Assume - # a main project A and a dependency B, which use different versions - # of Versioneer. A's setup.py imports A's Versioneer, leaving it in - # sys.modules by the time B's setup.py is executed, causing B to run - # with the wrong versioneer. Setuptools wraps the sub-dep builds in a - # sandbox that restores sys.modules to it's pre-build state, so the - # parent is protected against the child's "import versioneer". By - # removing ourselves from sys.modules here, before the child build - # happens, we protect the child from the parent's versioneer too. - # Also see https://github.com/warner/python-versioneer/issues/52 - - cmds = {} - - # we add "version" to both distutils and setuptools - from distutils.core import Command - - class cmd_version(Command): - description = "report generated version string" - user_options = [] - boolean_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - vers = get_versions(verbose=True) - print("Version: %s" % vers["version"]) - print(" full-revisionid: %s" % vers.get("full-revisionid")) - print(" dirty: %s" % vers.get("dirty")) - print(" date: %s" % vers.get("date")) - if vers["error"]: - print(" error: %s" % vers["error"]) - - cmds["version"] = cmd_version - - # we override "build_py" in both distutils and setuptools - # - # most invocation pathways end up running build_py: - # distutils/build -> build_py - # distutils/install -> distutils/build ->.. - # setuptools/bdist_wheel -> distutils/install ->.. - # setuptools/bdist_egg -> distutils/install_lib -> build_py - # setuptools/install -> bdist_egg ->.. - # setuptools/develop -> ? - # pip install: - # copies source tree to a tempdir before running egg_info/etc - # if .git isn't copied too, 'git describe' will fail - # then does setup.py bdist_wheel, or sometimes setup.py install - # setup.py egg_info -> ? - - # we override different "build_py" commands for both environments - if "setuptools" in sys.modules: - from setuptools.command.build_py import build_py as _build_py - else: - from distutils.command.build_py import build_py as _build_py - - class cmd_build_py(_build_py): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - _build_py.run(self) - # now locate _version.py in the new build/ directory and replace - # it with an updated value - if cfg.versionfile_build: - target_versionfile = os.path.join( - self.build_lib, cfg.versionfile_build - ) - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - cmds["build_py"] = cmd_build_py - - if "cx_Freeze" in sys.modules: # cx_freeze enabled? - from cx_Freeze.dist import build_exe as _build_exe - - # nczeczulin reports that py2exe won't like the pep440-style string - # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. - # setup(console=[{ - # "version": versioneer.get_version().split("+", 1)[0], # FILEVERSION - # "product_version": versioneer.get_version(), - # ... - - class cmd_build_exe(_build_exe): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - target_versionfile = cfg.versionfile_source - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - _build_exe.run(self) - os.unlink(target_versionfile) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - cmds["build_exe"] = cmd_build_exe - del cmds["build_py"] - - if "py2exe" in sys.modules: # py2exe enabled? - try: - from py2exe.distutils_buildexe import py2exe as _py2exe # py3 - except ImportError: - from py2exe.build_exe import py2exe as _py2exe # py2 - - class cmd_py2exe(_py2exe): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - target_versionfile = cfg.versionfile_source - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - _py2exe.run(self) - os.unlink(target_versionfile) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - cmds["py2exe"] = cmd_py2exe - - # we override different "sdist" commands for both environments - if "setuptools" in sys.modules: - from setuptools.command.sdist import sdist as _sdist - else: - from distutils.command.sdist import sdist as _sdist - - class cmd_sdist(_sdist): - def run(self): - versions = get_versions() - self._versioneer_generated_versions = versions - # unless we update this, the command will keep using the old - # version - self.distribution.metadata.version = versions["version"] - return _sdist.run(self) - - def make_release_tree(self, base_dir, files): - root = get_root() - cfg = get_config_from_root(root) - _sdist.make_release_tree(self, base_dir, files) - # now locate _version.py in the new base_dir directory - # (remembering that it may be a hardlink) and replace it with an - # updated value - target_versionfile = os.path.join(base_dir, cfg.versionfile_source) - print("UPDATING %s" % target_versionfile) - write_to_version_file( - target_versionfile, self._versioneer_generated_versions - ) - - cmds["sdist"] = cmd_sdist - - return cmds - - -CONFIG_ERROR = """ -setup.cfg is missing the necessary Versioneer configuration. You need -a section like: - - [versioneer] - VCS = git - style = pep440 - versionfile_source = src/myproject/_version.py - versionfile_build = myproject/_version.py - tag_prefix = - parentdir_prefix = myproject- - -You will also need to edit your setup.py to use the results: - - import versioneer - setup(version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), ...) - -Please read the docstring in ./versioneer.py for configuration instructions, -edit setup.cfg, and re-run the installer or 'python versioneer.py setup'. -""" - -SAMPLE_CONFIG = """ -# See the docstring in versioneer.py for instructions. Note that you must -# re-run 'versioneer.py setup' after changing this section, and commit the -# resulting files. - -[versioneer] -#VCS = git -#style = pep440 -#versionfile_source = -#versionfile_build = -#tag_prefix = -#parentdir_prefix = - -""" - -INIT_PY_SNIPPET = """ -from ._version import get_versions -__version__ = get_versions()['version'] -del get_versions -""" - - -def do_setup(): - """Main VCS-independent setup function for installing Versioneer.""" - root = get_root() - try: - cfg = get_config_from_root(root) - except ( - EnvironmentError, - configparser.NoSectionError, - configparser.NoOptionError, - ) as e: - if isinstance(e, (EnvironmentError, configparser.NoSectionError)): - print( - "Adding sample versioneer config to setup.cfg", file=sys.stderr - ) - with open(os.path.join(root, "setup.cfg"), "a") as f: - f.write(SAMPLE_CONFIG) - print(CONFIG_ERROR, file=sys.stderr) - return 1 - - print(" creating %s" % cfg.versionfile_source) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - ipy = os.path.join(os.path.dirname(cfg.versionfile_source), "__init__.py") - if os.path.exists(ipy): - try: - with open(ipy, "r") as f: - old = f.read() - except EnvironmentError: - old = "" - if INIT_PY_SNIPPET not in old: - print(" appending to %s" % ipy) - with open(ipy, "a") as f: - f.write(INIT_PY_SNIPPET) - else: - print(" %s unmodified" % ipy) - else: - print(" %s doesn't exist, ok" % ipy) - ipy = None - - # Make sure both the top-level "versioneer.py" and versionfile_source - # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so - # they'll be copied into source distributions. Pip won't be able to - # install the package without this. - manifest_in = os.path.join(root, "MANIFEST.in") - simple_includes = set() - try: - with open(manifest_in, "r") as f: - for line in f: - if line.startswith("include "): - for include in line.split()[1:]: - simple_includes.add(include) - except EnvironmentError: - pass - # That doesn't cover everything MANIFEST.in can do - # (http://docs.python.org/2/distutils/sourcedist.html#commands), so - # it might give some false negatives. Appending redundant 'include' - # lines is safe, though. - if "versioneer.py" not in simple_includes: - print(" appending 'versioneer.py' to MANIFEST.in") - with open(manifest_in, "a") as f: - f.write("include versioneer.py\n") - else: - print(" 'versioneer.py' already in MANIFEST.in") - if cfg.versionfile_source not in simple_includes: - print( - " appending versionfile_source ('%s') to MANIFEST.in" - % cfg.versionfile_source - ) - with open(manifest_in, "a") as f: - f.write("include %s\n" % cfg.versionfile_source) - else: - print(" versionfile_source already in MANIFEST.in") - - # Make VCS-specific changes. For git, this means creating/changing - # .gitattributes to mark _version.py for export-subst keyword - # substitution. - do_vcs_install(manifest_in, cfg.versionfile_source, ipy) - return 0 - - -def scan_setup_py(): - """Validate the contents of setup.py against Versioneer's expectations.""" - found = set() - setters = False - errors = 0 - with open("setup.py", "r") as f: - for line in f.readlines(): - if "import versioneer" in line: - found.add("import") - if "versioneer.get_cmdclass()" in line: - found.add("cmdclass") - if "versioneer.get_version()" in line: - found.add("get_version") - if "versioneer.VCS" in line: - setters = True - if "versioneer.versionfile_source" in line: - setters = True - if len(found) != 3: - print("") - print("Your setup.py appears to be missing some important items") - print("(but I might be wrong). Please make sure it has something") - print("roughly like the following:") - print("") - print(" import versioneer") - print(" setup( version=versioneer.get_version(),") - print(" cmdclass=versioneer.get_cmdclass(), ...)") - print("") - errors += 1 - if setters: - print("You should remove lines like 'versioneer.VCS = ' and") - print("'versioneer.versionfile_source = ' . This configuration") - print("now lives in setup.cfg, and should be removed from setup.py") - print("") - errors += 1 - return errors - - -if __name__ == "__main__": - cmd = sys.argv[1] - if cmd == "setup": - errors = do_setup() - errors += scan_setup_py() - if errors: - sys.exit(1) From 814cd161c604367ce9503153b6924d48ec12fb79 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 21 Sep 2021 17:07:22 -0500 Subject: [PATCH 230/519] chore: blacken with 19.10b0 to match shared templates (#386) * chore: blacken with 19.10b0 to match shared templates * remove unused versioneer file --- packages/pandas-gbq/.gitignore | 132 ++-- packages/pandas-gbq/.pre-commit-config.yaml | 31 + packages/pandas-gbq/.stickler.yml | 8 - packages/pandas-gbq/codecov.yml | 15 - packages/pandas-gbq/docs/source/conf.py | 17 +- packages/pandas-gbq/noxfile.py | 63 +- packages/pandas-gbq/pandas_gbq/_version.py | 571 ------------------ packages/pandas-gbq/pandas_gbq/auth.py | 7 +- packages/pandas-gbq/pandas_gbq/features.py | 12 +- packages/pandas-gbq/pandas_gbq/gbq.py | 74 +-- packages/pandas-gbq/pandas_gbq/load.py | 5 +- packages/pandas-gbq/pandas_gbq/schema.py | 13 +- packages/pandas-gbq/pyproject.toml | 8 - packages/pandas-gbq/samples/__init__.py | 1 - packages/pandas-gbq/samples/tests/__init__.py | 1 - packages/pandas-gbq/tests/__init__.py | 1 - packages/pandas-gbq/tests/system/__init__.py | 1 - packages/pandas-gbq/tests/system/conftest.py | 8 +- packages/pandas-gbq/tests/system/test_auth.py | 8 +- packages/pandas-gbq/tests/system/test_gbq.py | 215 ++----- .../system/test_read_gbq_with_bqstorage.py | 5 +- .../pandas-gbq/tests/system/test_to_gbq.py | 8 +- packages/pandas-gbq/tests/unit/__init__.py | 1 - packages/pandas-gbq/tests/unit/test_auth.py | 12 +- .../pandas-gbq/tests/unit/test_features.py | 4 +- packages/pandas-gbq/tests/unit/test_gbq.py | 108 +--- packages/pandas-gbq/tests/unit/test_load.py | 20 +- packages/pandas-gbq/tests/unit/test_schema.py | 24 +- .../pandas-gbq/tests/unit/test_timestamp.py | 10 +- 29 files changed, 271 insertions(+), 1112 deletions(-) create mode 100644 packages/pandas-gbq/.pre-commit-config.yaml delete mode 100644 packages/pandas-gbq/.stickler.yml delete mode 100644 packages/pandas-gbq/codecov.yml delete mode 100644 packages/pandas-gbq/pandas_gbq/_version.py delete mode 100644 packages/pandas-gbq/pyproject.toml diff --git a/packages/pandas-gbq/.gitignore b/packages/pandas-gbq/.gitignore index 5427c785f17f..53f230ec0d56 100644 --- a/packages/pandas-gbq/.gitignore +++ b/packages/pandas-gbq/.gitignore @@ -1,94 +1,64 @@ -######################################### -# Editor temporary/working/backup files # -.#* -*\#*\# -[#]*# -*~ -*$ -*.bak -*flymake* -*.kdev4 -*.log -*.swp -*.pdb -.project -.pydevproject -.settings -.idea -.vagrant -.noseids -.ipynb_checkpoints -.tags -.pytest_cache -.testmon* -.vscode/ -.env - -# Docs # -######## -docs/source/_build +*.py[cod] +*.sw[op] -# Coverage # -############ -.coverage -coverage.xml -coverage_html_report -.pytest_cache - -# Compiled source # -################### -*.a -*.com -*.class -*.dll -*.exe -*.pxi -*.o -*.py[ocd] +# C extensions *.so -.build_cache_dir -MANIFEST -__pycache__ -# Python files # -################ -# setup.py working directory -build -# setup.py dist directory -dist -# Egg metadata +# Packages +*.egg *.egg-info +dist +build +eggs .eggs -.pypirc +parts +bin +var +sdist +develop-eggs +.installed.cfg +lib +lib64 +__pycache__ -# tox testing tool -.tox -# rope -.ropeproject -# wheel files -*.whl -**/wheelhouse/* -pip-wheel-metadata +# Installer logs +pip-log.txt -# coverage +# Unit test / coverage reports .coverage -.testmondata -.pytest_cache .nox +.cache +.pytest_cache + -# OS generated files # -###################### -.directory -.gdb_history +# Mac .DS_Store -ehthumbs.db -Icon? -Thumbs.db -# caches # -.cache +# JetBrains +.idea + +# VS Code +.vscode + +# emacs +*~ + +# Built documentation +docs/_build +docs/source/_build +bigquery/docs/generated +docs.metadata + +# Virtual environment +env/ + +# Test logs +coverage.xml +*sponge_log.xml + +# System test environment variables. +system_tests/local_test_setup -# Credentials # -############### -bigquery_credentials.dat -ci/service_account.json +# Make sure a generated file isn't accidentally committed. +pylintrc +pylintrc.test diff --git a/packages/pandas-gbq/.pre-commit-config.yaml b/packages/pandas-gbq/.pre-commit-config.yaml new file mode 100644 index 000000000000..62eb5a77d9a3 --- /dev/null +++ b/packages/pandas-gbq/.pre-commit-config.yaml @@ -0,0 +1,31 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.0.1 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml +- repo: https://github.com/psf/black + rev: 19.10b0 + hooks: + - id: black +- repo: https://gitlab.com/pycqa/flake8 + rev: 3.9.2 + hooks: + - id: flake8 diff --git a/packages/pandas-gbq/.stickler.yml b/packages/pandas-gbq/.stickler.yml deleted file mode 100644 index 7bb34d25b67f..000000000000 --- a/packages/pandas-gbq/.stickler.yml +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright (c) 2017 pandas-gbq Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -linters: - black: - config: ./pyproject.toml - fixer: true \ No newline at end of file diff --git a/packages/pandas-gbq/codecov.yml b/packages/pandas-gbq/codecov.yml deleted file mode 100644 index 4c2ed9b16e4a..000000000000 --- a/packages/pandas-gbq/codecov.yml +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright (c) 2017 pandas-gbq Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -coverage: - status: - project: - default: - target: '0' - enabled: no - patch: - default: - enabled: no - target: '50' - branches: null diff --git a/packages/pandas-gbq/docs/source/conf.py b/packages/pandas-gbq/docs/source/conf.py index bfcc94efe87d..b250e7d088d0 100644 --- a/packages/pandas-gbq/docs/source/conf.py +++ b/packages/pandas-gbq/docs/source/conf.py @@ -70,9 +70,7 @@ # General information about the project. project = u"pandas-gbq" -copyright = u"2017-{}, PyData Development Team".format( - datetime.datetime.now().year -) +copyright = u"2017-{}, PyData Development Team".format(datetime.datetime.now().year) author = u"PyData Development Team" # The version info for the project you're documenting, acts as replacement for @@ -102,8 +100,13 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -# This patterns also effect to html_static_path and html_extra_path -exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] +exclude_patterns = [ + "_build", + "**/.nox/**/*", + "samples/AUTHORING_GUIDE.md", + "samples/CONTRIBUTING.md", + "samples/snippets/README.rst", +] # The reST default role (used for this markup: `text`) to use for all # documents. @@ -335,9 +338,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, "pandas-gbq", u"pandas-gbq Documentation", [author], 1) -] +man_pages = [(master_doc, "pandas-gbq", u"pandas-gbq Documentation", [author], 1)] # If true, show URL addresses after external links. # diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index e1564138fc38..b8a3a985648f 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -14,30 +14,48 @@ import nox -supported_pythons = ["3.7", "3.8"] -system_test_pythons = ["3.7", "3.8"] -latest_python = "3.8" +BLACK_VERSION = "black==19.10b0" +BLACK_PATHS = ["docs", "pandas_gbq", "tests", "noxfile.py", "setup.py"] -# Use a consistent version of black so CI is deterministic. -# Should match Stickler: https://stickler-ci.com/docs#black -black_package = "black==20.8b1" +DEFAULT_PYTHON_VERSION = "3.8" +SYSTEM_TEST_PYTHON_VERSIONS = ["3.8"] +UNIT_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9"] -@nox.session(python=latest_python) +# Error if a python version is missing +nox.options.error_on_missing_interpreters = True + + +@nox.session(python=DEFAULT_PYTHON_VERSION) def lint(session): - session.install(black_package, "flake8") - session.run("flake8", "pandas_gbq") - session.run("flake8", "tests") - session.run("black", "--check", ".") + """Run linters. + Returns a failure if the linters find linting errors or sufficiently + serious code quality issues. + """ + session.install("flake8", BLACK_VERSION) + session.run( + "black", "--check", *BLACK_PATHS, + ) + session.run("flake8", "pandas_gbq", "tests") -@nox.session(python=latest_python) +@nox.session(python=DEFAULT_PYTHON_VERSION) def blacken(session): - session.install(black_package) - session.run("black", ".") + """Run black. Format code to uniform standard.""" + session.install(BLACK_VERSION) + session.run( + "black", *BLACK_PATHS, + ) + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def lint_setup_py(session): + """Verify that setup.py is valid (including RST check).""" + session.install("docutils", "pygments") + session.run("python", "setup.py", "check", "--restructuredtext", "--strict") -@nox.session(python=supported_pythons) + +@nox.session(python=UNIT_TEST_PYTHON_VERSIONS) def unit(session): session.install("pytest", "pytest-cov") session.install( @@ -56,18 +74,23 @@ def unit(session): "--cov=tests.unit", "--cov-report", "xml:/tmp/pytest-cov.xml", - *session.posargs + *session.posargs, ) -@nox.session(python=latest_python) +@nox.session(python=DEFAULT_PYTHON_VERSION) def cover(session): + """Run the final coverage report. + This outputs the coverage report aggregating coverage from the unit + test runs (not system test runs), and then erases coverage data. + """ session.install("coverage", "pytest-cov") session.run("coverage", "report", "--show-missing", "--fail-under=73") + session.run("coverage", "erase") -@nox.session(python=latest_python) +@nox.session(python=DEFAULT_PYTHON_VERSION) def docs(session): """Build the docs.""" @@ -89,7 +112,7 @@ def docs(session): ) -@nox.session(python=system_test_pythons) +@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) def system(session): session.install("pytest", "pytest-cov") session.install( @@ -111,5 +134,5 @@ def system(session): os.path.join(".", "tests", "system"), os.path.join(".", "samples", "tests"), "-v", - *additional_args + *additional_args, ) diff --git a/packages/pandas-gbq/pandas_gbq/_version.py b/packages/pandas-gbq/pandas_gbq/_version.py deleted file mode 100644 index 017eefdf9694..000000000000 --- a/packages/pandas-gbq/pandas_gbq/_version.py +++ /dev/null @@ -1,571 +0,0 @@ -# Copyright (c) 2017 pandas-gbq Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - - -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (built by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. - -# This file is released into the public domain. Generated by -# versioneer-0.18 (https://github.com/warner/python-versioneer) - -"""Git implementation of _version.py.""" - -import errno -import os -import re -import subprocess -import sys - - -def get_keywords(): - """Get the keywords needed to look up the version information.""" - # these strings will be replaced by git during git-archive. - # setup.py/versioneer.py will grep for the variable names, so they must - # each be defined on a line of their own. _version.py will just call - # get_keywords(). - git_refnames = "$Format:%d$" - git_full = "$Format:%H$" - git_date = "$Format:%ci$" - keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} - return keywords - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_config(): - """Create, populate and return the VersioneerConfig() object.""" - # these strings are filled in when 'setup.py versioneer' creates - # _version.py - cfg = VersioneerConfig() - cfg.VCS = "git" - cfg.style = "pep440" - cfg.tag_prefix = "" - cfg.parentdir_prefix = "pandas_gbq/_version.py" - cfg.versionfile_source = "pandas_gbq/_version.py" - cfg.verbose = False - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - - return decorate - - -def run_command( - commands, args, cwd=None, verbose=False, hide_stderr=False, env=None -): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen( - [c] + args, - cwd=cwd, - env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr else None), - ) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %s" % (commands,)) - return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %s (error)" % dispcmd) - print("stdout was %s" % stdout) - return None, p.returncode - return stdout, p.returncode - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for i in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return { - "version": dirname[len(parentdir_prefix) :], - "full-revisionid": None, - "dirty": False, - "error": None, - "date": None, - } - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print( - "Tried directories %s but none started with prefix %s" - % (str(rootdirs), parentdir_prefix) - ) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - date = keywords.get("date") - if date is not None: - # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r"\d", r)]) - if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix) :] - if verbose: - print("picking %s" % r) - return { - "version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": None, - "date": date, - } - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return { - "version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": "no suitable tags", - "date": None, - } - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - out, rc = run_command( - GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True - ) - if rc != 0: - if verbose: - print("Directory %s not under git control" % root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command( - GITS, - [ - "describe", - "--tags", - "--dirty", - "--always", - "--long", - "--match", - "%s*" % tag_prefix, - ], - cwd=root, - ) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[: git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ( - "unable to parse git-describe output: '%s'" % describe_out - ) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%s' doesn't start with prefix '%s'" - print(fmt % (full_tag, tag_prefix)) - pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( - full_tag, - tag_prefix, - ) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix) :] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out, rc = run_command( - GITS, ["rev-list", "HEAD", "--count"], cwd=root - ) - pieces["distance"] = int(count_out) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ - 0 - ].strip() - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%d" % pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%d" % pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Eexceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return { - "version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None, - } - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%s'" % style) - - return { - "version": rendered, - "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], - "error": None, - "date": pieces.get("date"), - } - - -def get_versions(): - """Get version information or return default if unable to do so.""" - # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have - # __file__, we can work backwards from there to the root. Some - # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded keywords. - - cfg = get_config() - verbose = cfg.verbose - - try: - return git_versions_from_keywords( - get_keywords(), cfg.tag_prefix, verbose - ) - except NotThisMethod: - pass - - try: - root = os.path.realpath(__file__) - # versionfile_source is the relative path from the top of the source - # tree (where the .git directory might live) to this file. Invert - # this to find the root from __file__. - for i in cfg.versionfile_source.split("/"): - root = os.path.dirname(root) - except NameError: - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None, - } - - try: - pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) - return render(pieces, cfg.style) - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - except NotThisMethod: - pass - - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", - "date": None, - } diff --git a/packages/pandas-gbq/pandas_gbq/auth.py b/packages/pandas-gbq/pandas_gbq/auth.py index 61dcee06ffa4..41ee4192f573 100644 --- a/packages/pandas-gbq/pandas_gbq/auth.py +++ b/packages/pandas-gbq/pandas_gbq/auth.py @@ -23,9 +23,7 @@ # machine. # # See: https://cloud.google.com/docs/authentication/end-user for details. -CLIENT_ID = ( - "725825577420-unm2gnkiprugilg743tkbig250f4sfsj.apps.googleusercontent.com" -) +CLIENT_ID = "725825577420-unm2gnkiprugilg743tkbig250f4sfsj.apps.googleusercontent.com" CLIENT_SECRET = "4hqze9yI8fxShls8eJWkeMdJ" @@ -60,8 +58,7 @@ def get_credentials_cache(reauth): if reauth: return pydata_google_auth.cache.WriteOnlyCredentialsCache( - dirname=CREDENTIALS_CACHE_DIRNAME, - filename=CREDENTIALS_CACHE_FILENAME, + dirname=CREDENTIALS_CACHE_DIRNAME, filename=CREDENTIALS_CACHE_FILENAME, ) return pydata_google_auth.cache.ReadWriteCredentialsCache( dirname=CREDENTIALS_CACHE_DIRNAME, filename=CREDENTIALS_CACHE_FILENAME diff --git a/packages/pandas-gbq/pandas_gbq/features.py b/packages/pandas-gbq/pandas_gbq/features.py index 5a90caa233c3..ef1969fd6407 100644 --- a/packages/pandas-gbq/pandas_gbq/features.py +++ b/packages/pandas-gbq/pandas_gbq/features.py @@ -28,9 +28,7 @@ def bigquery_installed_version(self): self._bigquery_installed_version = pkg_resources.parse_version( google.cloud.bigquery.__version__ ) - bigquery_minimum_version = pkg_resources.parse_version( - BIGQUERY_MINIMUM_VERSION - ) + bigquery_minimum_version = pkg_resources.parse_version(BIGQUERY_MINIMUM_VERSION) if self._bigquery_installed_version < bigquery_minimum_version: raise ImportError( @@ -67,9 +65,7 @@ def bigquery_has_from_dataframe_with_csv(self): bigquery_from_dataframe_version = pkg_resources.parse_version( BIGQUERY_FROM_DATAFRAME_CSV_VERSION ) - return ( - self.bigquery_installed_version >= bigquery_from_dataframe_version - ) + return self.bigquery_installed_version >= bigquery_from_dataframe_version @property def pandas_installed_version(self): @@ -79,9 +75,7 @@ def pandas_installed_version(self): if self._pandas_installed_version is not None: return self._pandas_installed_version - self._pandas_installed_version = pkg_resources.parse_version( - pandas.__version__ - ) + self._pandas_installed_version = pkg_resources.parse_version(pandas.__version__) return self._pandas_installed_version @property diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index e7f8b0ae4ed0..856c128529c6 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -309,9 +309,7 @@ def __init__( self.project_id = default_project if self.project_id is None: - raise ValueError( - "Could not determine project ID and one was not supplied." - ) + raise ValueError("Could not determine project ID and one was not supplied.") # Cache the credentials if they haven't been set yet. if context.credentials is None: @@ -372,9 +370,7 @@ def get_client(self): client_info=client_info, ) - return bigquery.Client( - project=self.project_id, credentials=self.credentials - ) + return bigquery.Client(project=self.project_id, credentials=self.credentials) @staticmethod def process_http_error(ex): @@ -383,9 +379,7 @@ def process_http_error(ex): raise GenericGBQException("Reason: {0}".format(ex)) - def run_query( - self, query, max_results=None, progress_bar_type=None, **kwargs - ): + def run_query(self, query, max_results=None, progress_bar_type=None, **kwargs): from concurrent.futures import TimeoutError from google.auth.exceptions import RefreshError @@ -423,9 +417,7 @@ def run_query( logger.debug("Query running...") except (RefreshError, ValueError): if self.private_key: - raise AccessDenied( - "The service account credentials are not valid" - ) + raise AccessDenied("The service account credentials are not valid") else: raise AccessDenied( "The credentials have been revoked or expired, " @@ -440,9 +432,9 @@ def run_query( while query_reply.state != "DONE": self.log_elapsed_seconds(" Elapsed", "s. Waiting...") - timeout_ms = job_config.get("jobTimeoutMs") or job_config[ - "query" - ].get("timeoutMs") + timeout_ms = job_config.get("jobTimeoutMs") or job_config["query"].get( + "timeoutMs" + ) timeout_ms = int(timeout_ms) if timeout_ms else None if timeout_ms and timeout_ms < self.get_elapsed_seconds() * 1000: raise QueryTimeout("Query timeout: {} ms".format(timeout_ms)) @@ -467,8 +459,7 @@ def run_query( bytes_billed = query_reply.total_bytes_billed or 0 logger.debug( "Query done.\nProcessed: {} Billed: {}".format( - self.sizeof_fmt(bytes_processed), - self.sizeof_fmt(bytes_billed), + self.sizeof_fmt(bytes_processed), self.sizeof_fmt(bytes_billed), ) ) logger.debug( @@ -486,11 +477,7 @@ def run_query( ) def _download_results( - self, - query_job, - max_results=None, - progress_bar_type=None, - user_dtypes=None, + self, query_job, max_results=None, progress_bar_type=None, user_dtypes=None, ): # No results are desired, so don't bother downloading anything. if max_results == 0: @@ -519,17 +506,13 @@ def _download_results( to_dataframe_kwargs = {} if FEATURES.bigquery_has_bqstorage: - to_dataframe_kwargs[ - "create_bqstorage_client" - ] = create_bqstorage_client + to_dataframe_kwargs["create_bqstorage_client"] = create_bqstorage_client try: query_job.result() # Get the table schema, so that we can list rows. destination = self.client.get_table(query_job.destination) - rows_iter = self.client.list_rows( - destination, max_results=max_results - ) + rows_iter = self.client.list_rows(destination, max_results=max_results) schema_fields = [field.to_api_repr() for field in rows_iter.schema] conversion_dtypes = _bqschema_to_nullsafe_dtypes(schema_fields) @@ -584,9 +567,7 @@ def load_data( self.process_http_error(ex) def delete_and_recreate_table(self, dataset_id, table_id, table_schema): - table = _Table( - self.project_id, dataset_id, credentials=self.credentials - ) + table = _Table(self.project_id, dataset_id, credentials=self.credentials) table.delete(table_id) table.create(table_id, table_schema) @@ -644,9 +625,7 @@ def _cast_empty_df_dtypes(schema_fields, df): ``object``. """ if not df.empty: - raise ValueError( - "DataFrame must be empty in order to cast non-nullsafe dtypes" - ) + raise ValueError("DataFrame must be empty in order to cast non-nullsafe dtypes") dtype_map = {"BOOLEAN": bool, "INTEGER": np.int64} @@ -867,9 +846,7 @@ def read_gbq( final_df.set_index(index_col, inplace=True) else: raise InvalidIndexColumn( - 'Index column "{0}" does not exist in DataFrame.'.format( - index_col - ) + 'Index column "{0}" does not exist in DataFrame.'.format(index_col) ) # Change the order of columns in the DataFrame based on provided list @@ -877,9 +854,7 @@ def read_gbq( if sorted(col_order) == sorted(final_df.columns): final_df = final_df[col_order] else: - raise InvalidColumnOrder( - "Column order does not match this DataFrame." - ) + raise InvalidColumnOrder("Column order does not match this DataFrame.") connector.log_elapsed_seconds( "Total time taken", @@ -1070,13 +1045,9 @@ def to_gbq( "'append' or 'replace' data." ) elif if_exists == "replace": - connector.delete_and_recreate_table( - dataset_id, table_id, table_schema - ) + connector.delete_and_recreate_table(dataset_id, table_id, table_schema) elif if_exists == "append": - if not pandas_gbq.schema.schema_is_subset( - original_schema, table_schema - ): + if not pandas_gbq.schema.schema_is_subset(original_schema, table_schema): raise InvalidSchema( "Please verify that the structure and " "data types in the DataFrame match the " @@ -1116,8 +1087,7 @@ def generate_bq_schema(df, default_type="STRING"): """ # deprecation TimeSeries, #11121 warnings.warn( - "generate_bq_schema is deprecated and will be removed in " - "a future version", + "generate_bq_schema is deprecated and will be removed in " "a future version", FutureWarning, stacklevel=2, ) @@ -1206,17 +1176,13 @@ def create(self, table_id, schema): from google.cloud.bigquery import TableReference if self.exists(table_id): - raise TableCreationError( - "Table {0} already exists".format(table_id) - ) + raise TableCreationError("Table {0} already exists".format(table_id)) if not _Dataset(self.project_id, credentials=self.credentials).exists( self.dataset_id ): _Dataset( - self.project_id, - credentials=self.credentials, - location=self.location, + self.project_id, credentials=self.credentials, location=self.location, ).create(self.dataset_id) table_ref = TableReference( diff --git a/packages/pandas-gbq/pandas_gbq/load.py b/packages/pandas-gbq/pandas_gbq/load.py index 4eca7c560655..faa674c21d5c 100644 --- a/packages/pandas-gbq/pandas_gbq/load.py +++ b/packages/pandas-gbq/pandas_gbq/load.py @@ -81,10 +81,7 @@ def load_chunks( if FEATURES.bigquery_has_from_dataframe_with_csv: client.load_table_from_dataframe( - chunk, - destination_table_ref, - job_config=job_config, - location=location, + chunk, destination_table_ref, job_config=job_config, location=location, ).result() else: try: diff --git a/packages/pandas-gbq/pandas_gbq/schema.py b/packages/pandas-gbq/pandas_gbq/schema.py index ec81045c1559..e2f97455ecaf 100644 --- a/packages/pandas-gbq/pandas_gbq/schema.py +++ b/packages/pandas-gbq/pandas_gbq/schema.py @@ -21,9 +21,7 @@ def to_pandas_gbq(client_schema): """Given a sequence of :class:`google.cloud.bigquery.schema.SchemaField`, return a schema in pandas-gbq API format. """ - remote_fields = [ - field_remote.to_api_repr() for field_remote in client_schema - ] + remote_fields = [field_remote.to_api_repr() for field_remote in client_schema] for field in remote_fields: field["type"] = field["type"].upper() field["mode"] = field["mode"].upper() @@ -39,9 +37,7 @@ def to_google_cloud_bigquery(pandas_gbq_schema): # Need to convert from JSON representation to format used by client library. schema = add_default_nullable_mode(pandas_gbq_schema) - return [ - bigquery.SchemaField.from_api_repr(field) for field in schema["fields"] - ] + return [bigquery.SchemaField.from_api_repr(field) for field in schema["fields"]] def _clean_schema_fields(fields): @@ -110,10 +106,7 @@ def generate_bq_schema(dataframe, default_type="STRING"): fields = [] for column_name, dtype in dataframe.dtypes.iteritems(): fields.append( - { - "name": column_name, - "type": type_mapping.get(dtype.kind, default_type), - } + {"name": column_name, "type": type_mapping.get(dtype.kind, default_type)} ) return {"fields": fields} diff --git a/packages/pandas-gbq/pyproject.toml b/packages/pandas-gbq/pyproject.toml deleted file mode 100644 index 318a04420365..000000000000 --- a/packages/pandas-gbq/pyproject.toml +++ /dev/null @@ -1,8 +0,0 @@ -[tool.black] -line-length = 79 -exclude = ''' -versioneer.py -| _version.py -| docs -| .nox -''' \ No newline at end of file diff --git a/packages/pandas-gbq/samples/__init__.py b/packages/pandas-gbq/samples/__init__.py index edbca6c3c489..c9ab850639a6 100644 --- a/packages/pandas-gbq/samples/__init__.py +++ b/packages/pandas-gbq/samples/__init__.py @@ -1,4 +1,3 @@ # Copyright (c) 2017 pandas-gbq Authors All rights reserved. # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. - diff --git a/packages/pandas-gbq/samples/tests/__init__.py b/packages/pandas-gbq/samples/tests/__init__.py index edbca6c3c489..c9ab850639a6 100644 --- a/packages/pandas-gbq/samples/tests/__init__.py +++ b/packages/pandas-gbq/samples/tests/__init__.py @@ -1,4 +1,3 @@ # Copyright (c) 2017 pandas-gbq Authors All rights reserved. # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. - diff --git a/packages/pandas-gbq/tests/__init__.py b/packages/pandas-gbq/tests/__init__.py index edbca6c3c489..c9ab850639a6 100644 --- a/packages/pandas-gbq/tests/__init__.py +++ b/packages/pandas-gbq/tests/__init__.py @@ -1,4 +1,3 @@ # Copyright (c) 2017 pandas-gbq Authors All rights reserved. # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. - diff --git a/packages/pandas-gbq/tests/system/__init__.py b/packages/pandas-gbq/tests/system/__init__.py index edbca6c3c489..c9ab850639a6 100644 --- a/packages/pandas-gbq/tests/system/__init__.py +++ b/packages/pandas-gbq/tests/system/__init__.py @@ -1,4 +1,3 @@ # Copyright (c) 2017 pandas-gbq Authors All rights reserved. # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. - diff --git a/packages/pandas-gbq/tests/system/conftest.py b/packages/pandas-gbq/tests/system/conftest.py index a40ac47b4726..4745da0c7137 100644 --- a/packages/pandas-gbq/tests/system/conftest.py +++ b/packages/pandas-gbq/tests/system/conftest.py @@ -32,9 +32,7 @@ def gbq_connector(project, credentials): def random_dataset(bigquery_client, random_dataset_id): from google.cloud import bigquery - dataset_ref = bigquery.DatasetReference( - bigquery_client.project, random_dataset_id - ) + dataset_ref = bigquery.DatasetReference(bigquery_client.project, random_dataset_id) dataset = bigquery.Dataset(dataset_ref) bigquery_client.create_dataset(dataset) return dataset @@ -44,9 +42,7 @@ def random_dataset(bigquery_client, random_dataset_id): def tokyo_dataset(bigquery_client, random_dataset_id): from google.cloud import bigquery - dataset_ref = bigquery.DatasetReference( - bigquery_client.project, random_dataset_id - ) + dataset_ref = bigquery.DatasetReference(bigquery_client.project, random_dataset_id) dataset = bigquery.Dataset(dataset_ref) dataset.location = "asia-northeast1" bigquery_client.create_dataset(dataset) diff --git a/packages/pandas-gbq/tests/system/test_auth.py b/packages/pandas-gbq/tests/system/test_auth.py index 5e8f5a4755af..34d5c8ffd253 100644 --- a/packages/pandas-gbq/tests/system/test_auth.py +++ b/packages/pandas-gbq/tests/system/test_auth.py @@ -52,9 +52,7 @@ def _check_if_can_get_correct_default_credentials(): import pandas_gbq.gbq try: - credentials, project = google.auth.default( - scopes=pandas_gbq.auth.SCOPES - ) + credentials, project = google.auth.default(scopes=pandas_gbq.auth.SCOPES) except (DefaultCredentialsError, IOError): return False @@ -68,9 +66,7 @@ def test_should_be_able_to_get_valid_credentials(project_id, private_key_path): @pytest.mark.local_auth -def test_get_credentials_bad_file_returns_user_credentials( - project_id, monkeypatch -): +def test_get_credentials_bad_file_returns_user_credentials(project_id, monkeypatch): import google.auth from google.auth.credentials import Credentials diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index bbc0da642b2c..00bbd3d6b16c 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -27,9 +27,7 @@ TABLE_ID = "new_test" PANDAS_VERSION = pkg_resources.parse_version(pandas.__version__) NULLABLE_INT_PANDAS_VERSION = pkg_resources.parse_version("0.24.0") -NULLABLE_INT_MESSAGE = ( - "Require pandas 0.24+ in order to use nullable integer type." -) +NULLABLE_INT_MESSAGE = "Require pandas 0.24+ in order to use nullable integer type." def test_imports(): @@ -43,8 +41,7 @@ def make_mixed_dataframe_v2(test_size): ints = np.random.randint(1, 10, size=(1, test_size)) strs = np.random.randint(1, 10, size=(1, test_size)).astype(str) times = [ - datetime.datetime.now(pytz.timezone("US/Arizona")) - for t in range(test_size) + datetime.datetime.now(pytz.timezone("US/Arizona")) for t in range(test_size) ] return DataFrame( { @@ -58,9 +55,7 @@ def make_mixed_dataframe_v2(test_size): ) -def get_schema( - gbq_connector: gbq.GbqConnector, dataset_id: str, table_id: str -): +def get_schema(gbq_connector: gbq.GbqConnector, dataset_id: str, table_id: str): """Retrieve the schema of the table Obtain from BigQuery the field names and field types @@ -82,17 +77,14 @@ def get_schema( bqclient = gbq_connector.client table_ref = bigquery.TableReference( - bigquery.DatasetReference(bqclient.project, dataset_id), - table_id, + bigquery.DatasetReference(bqclient.project, dataset_id), table_id, ) try: table = bqclient.get_table(table_ref) remote_schema = table.schema - remote_fields = [ - field_remote.to_api_repr() for field_remote in remote_schema - ] + remote_fields = [field_remote.to_api_repr() for field_remote in remote_schema] for field in remote_fields: field["type"] = field["type"].upper() field["mode"] = field["mode"].upper() @@ -197,11 +189,7 @@ def test_should_properly_handle_nullable_integers(self, project_id): tm.assert_frame_equal( df, DataFrame( - { - "nullable_integer": pandas.Series( - [1, pandas.NA], dtype="Int64" - ) - } + {"nullable_integer": pandas.Series([1, pandas.NA], dtype="Int64")} ), ) @@ -232,11 +220,7 @@ def test_should_properly_handle_nullable_longs(self, project_id): tm.assert_frame_equal( df, DataFrame( - { - "nullable_long": pandas.Series( - [1 << 62, pandas.NA], dtype="Int64" - ) - } + {"nullable_long": pandas.Series([1 << 62, pandas.NA], dtype="Int64")} ), ) @@ -253,10 +237,7 @@ def test_should_properly_handle_null_integers(self, project_id): dtypes={"null_integer": "Int64"}, ) tm.assert_frame_equal( - df, - DataFrame( - {"null_integer": pandas.Series([pandas.NA], dtype="Int64")} - ), + df, DataFrame({"null_integer": pandas.Series([pandas.NA], dtype="Int64")}), ) def test_should_properly_handle_valid_floats(self, project_id): @@ -295,9 +276,7 @@ def test_should_properly_handle_valid_doubles(self, project_id): credentials=self.credentials, dialect="legacy", ) - tm.assert_frame_equal( - df, DataFrame({"valid_double": [pi * 10 ** 307]}) - ) + tm.assert_frame_equal(df, DataFrame({"valid_double": [pi * 10 ** 307]})) def test_should_properly_handle_nullable_doubles(self, project_id): from math import pi @@ -329,11 +308,7 @@ def test_should_properly_handle_null_floats(self, project_id): def test_should_properly_handle_date(self, project_id): query = "SELECT DATE(2003, 1, 4) AS date_col" - df = gbq.read_gbq( - query, - project_id=project_id, - credentials=self.credentials, - ) + df = gbq.read_gbq(query, project_id=project_id, credentials=self.credentials,) expected = DataFrame( { "date_col": pandas.Series( @@ -344,12 +319,10 @@ def test_should_properly_handle_date(self, project_id): tm.assert_frame_equal(df, expected) def test_should_properly_handle_time(self, project_id): - query = "SELECT TIME_ADD(TIME(3, 14, 15), INTERVAL 926589 MICROSECOND) AS time_col" - df = gbq.read_gbq( - query, - project_id=project_id, - credentials=self.credentials, + query = ( + "SELECT TIME_ADD(TIME(3, 14, 15), INTERVAL 926589 MICROSECOND) AS time_col" ) + df = gbq.read_gbq(query, project_id=project_id, credentials=self.credentials,) expected = DataFrame( { "time_col": pandas.Series( @@ -368,13 +341,10 @@ def test_should_properly_handle_timestamp_unix_epoch(self, project_id): dialect="legacy", ) expected = DataFrame( - {"unix_epoch": ["1970-01-01T00:00:00.000000Z"]}, - dtype="datetime64[ns]", + {"unix_epoch": ["1970-01-01T00:00:00.000000Z"]}, dtype="datetime64[ns]", ) if expected["unix_epoch"].dt.tz is None: - expected["unix_epoch"] = expected["unix_epoch"].dt.tz_localize( - "UTC" - ) + expected["unix_epoch"] = expected["unix_epoch"].dt.tz_localize("UTC") tm.assert_frame_equal(df, expected) def test_should_properly_handle_arbitrary_timestamp(self, project_id): @@ -390,9 +360,9 @@ def test_should_properly_handle_arbitrary_timestamp(self, project_id): dtype="datetime64[ns]", ) if expected["valid_timestamp"].dt.tz is None: - expected["valid_timestamp"] = expected[ - "valid_timestamp" - ].dt.tz_localize("UTC") + expected["valid_timestamp"] = expected["valid_timestamp"].dt.tz_localize( + "UTC" + ) tm.assert_frame_equal(df, expected) def test_should_properly_handle_datetime_unix_epoch(self, project_id): @@ -405,9 +375,7 @@ def test_should_properly_handle_datetime_unix_epoch(self, project_id): ) tm.assert_frame_equal( df, - DataFrame( - {"unix_epoch": ["1970-01-01T00:00:00"]}, dtype="datetime64[ns]" - ), + DataFrame({"unix_epoch": ["1970-01-01T00:00:00"]}, dtype="datetime64[ns]"), ) def test_should_properly_handle_arbitrary_datetime(self, project_id): @@ -419,10 +387,7 @@ def test_should_properly_handle_arbitrary_datetime(self, project_id): dialect="legacy", ) tm.assert_frame_equal( - df, - DataFrame( - {"valid_timestamp": [np.datetime64("2004-09-15T05:00:00")]} - ), + df, DataFrame({"valid_timestamp": [np.datetime64("2004-09-15T05:00:00")]}), ) @pytest.mark.parametrize( @@ -435,9 +400,7 @@ def test_should_properly_handle_arbitrary_datetime(self, project_id): ("FALSE", pandas.api.types.is_bool_dtype), ], ) - def test_return_correct_types( - self, project_id, expression, is_expected_dtype - ): + def test_return_correct_types(self, project_id, expression, is_expected_dtype): """ All type checks can be added to this function using additional parameters, rather than creating additional functions. @@ -464,9 +427,7 @@ def test_should_properly_handle_null_timestamp(self, project_id): dialect="legacy", ) expected = DataFrame({"null_timestamp": [NaT]}, dtype="datetime64[ns]") - expected["null_timestamp"] = expected["null_timestamp"].dt.tz_localize( - "UTC" - ) + expected["null_timestamp"] = expected["null_timestamp"].dt.tz_localize("UTC") tm.assert_frame_equal(df, expected) def test_should_properly_handle_null_datetime(self, project_id): @@ -525,9 +486,9 @@ def test_index_column(self, project_id): credentials=self.credentials, dialect="legacy", ) - correct_frame = DataFrame( - {"string_1": ["a"], "string_2": ["b"]} - ).set_index("string_1") + correct_frame = DataFrame({"string_1": ["a"], "string_2": ["b"]}).set_index( + "string_1" + ) assert result_frame.index.name == correct_frame.index.name def test_column_order(self, project_id): @@ -672,8 +633,7 @@ def test_zero_rows(self, project_id): "iso_time": pandas.Series([], dtype="datetime64[ns]"), } expected_result = DataFrame( - empty_columns, - columns=["name", "number", "is_hurricane", "iso_time"], + empty_columns, columns=["name", "number", "is_hurricane", "iso_time"], ) tm.assert_frame_equal(df, expected_result, check_index_type=False) @@ -712,8 +672,7 @@ def test_legacy_sql(self, project_id): def test_standard_sql(self, project_id): standard_sql = ( - "SELECT DISTINCT id FROM " - "`publicdata.samples.wikipedia` LIMIT 10" + "SELECT DISTINCT id FROM " "`publicdata.samples.wikipedia` LIMIT 10" ) # Test that a standard sql statement fails when using @@ -828,9 +787,7 @@ def test_configuration_without_query(self, project_id): dialect="legacy", ) - def test_configuration_raises_value_error_with_multiple_config( - self, project_id - ): + def test_configuration_raises_value_error_with_multiple_config(self, project_id): sql_statement = "SELECT 1" config = { "query": {"query": sql_statement, "useQueryCache": False}, @@ -900,8 +857,7 @@ def test_struct(self, project_id): dialect="standard", ) expected = DataFrame( - [[1, {"letter": "a", "num": 1}]], - columns=["int_field", "struct_field"], + [[1, {"letter": "a", "num": 1}]], columns=["int_field", "struct_field"], ) tm.assert_frame_equal(df, expected) @@ -914,8 +870,7 @@ def test_array(self, project_id): dialect="standard", ) tm.assert_frame_equal( - df, - DataFrame([[["a", "x", "b", "y", "c", "z"]]], columns=["letters"]), + df, DataFrame([[["a", "x", "b", "y", "c", "z"]]], columns=["letters"]), ) def test_array_length_zero(self, project_id): @@ -934,8 +889,7 @@ def test_array_length_zero(self, project_id): dialect="standard", ) expected = DataFrame( - [["a", [""], 1], ["b", [], 0]], - columns=["letter", "array_field", "len"], + [["a", [""], 1], ["b", [], 0]], columns=["letter", "array_field", "len"], ) tm.assert_frame_equal(df, expected) @@ -958,10 +912,7 @@ def test_array_agg(self, project_id): dialect="standard", ) tm.assert_frame_equal( - df, - DataFrame( - [["a", [1, 3]], ["b", [2]]], columns=["letter", "numbers"] - ), + df, DataFrame([["a", [1, 3]], ["b", [2]]], columns=["letter", "numbers"]), ) def test_array_of_floats(self, project_id): @@ -972,9 +923,7 @@ def test_array_of_floats(self, project_id): credentials=self.credentials, dialect="standard", ) - tm.assert_frame_equal( - df, DataFrame([[[1.1, 2.2, 3.3], 4]], columns=["a", "b"]) - ) + tm.assert_frame_equal(df, DataFrame([[[1.1, 2.2, 3.3], 4]], columns=["a", "b"])) def test_tokyo(self, tokyo_dataset, tokyo_table, project_id): df = gbq.read_gbq( @@ -998,9 +947,7 @@ def setup(self, project, credentials, random_dataset_id): self.credentials = credentials self.gbq_connector = gbq.GbqConnector(project, credentials=credentials) self.bqclient = self.gbq_connector.client - self.table = gbq._Table( - project, random_dataset_id, credentials=credentials - ) + self.table = gbq._Table(project, random_dataset_id, credentials=credentials) self.destination_table = "{}.{}".format(random_dataset_id, TABLE_ID) def test_upload_data(self, project_id): @@ -1044,10 +991,7 @@ def test_upload_empty_data(self, project_id): def test_upload_empty_data_with_schema(self, project_id): test_id = "data_with_0_rows" df = DataFrame( - { - "a": pandas.Series(dtype="int64"), - "b": pandas.Series(dtype="object"), - } + {"a": pandas.Series(dtype="int64"), "b": pandas.Series(dtype="object")} ) gbq.to_gbq( @@ -1244,9 +1188,7 @@ def test_google_upload_errors_should_raise_exception(self, project_id): def test_upload_chinese_unicode_data(self, project_id): test_id = "2" test_size = 6 - df = DataFrame( - np.random.randn(6, 4), index=range(6), columns=list("ABCD") - ) + df = DataFrame(np.random.randn(6, 4), index=range(6), columns=list("ABCD")) df["s"] = u"信用卡" gbq.to_gbq( @@ -1419,9 +1361,7 @@ def test_upload_data_with_valid_user_schema(self, project_id): self.gbq_connector, dataset, table, dict(fields=test_schema) ) - def test_upload_data_with_invalid_user_schema_raises_error( - self, project_id - ): + def test_upload_data_with_invalid_user_schema_raises_error(self, project_id): df = tm.makeMixedDataFrame() test_id = "19" test_schema = [ @@ -1440,9 +1380,7 @@ def test_upload_data_with_invalid_user_schema_raises_error( table_schema=test_schema, ) - def test_upload_data_with_missing_schema_fields_raises_error( - self, project_id - ): + def test_upload_data_with_missing_schema_fields_raises_error(self, project_id): df = tm.makeMixedDataFrame() test_id = "20" test_schema = [ @@ -1464,9 +1402,7 @@ def test_upload_data_with_timestamp(self, project_id): test_id = "21" test_size = 6 df = DataFrame( - np.random.randn(test_size, 4), - index=range(test_size), - columns=list("ABCD"), + np.random.randn(test_size, 4), index=range(test_size), columns=list("ABCD"), ) df["times"] = pandas.Series( [ @@ -1524,9 +1460,7 @@ def test_upload_data_with_different_df_and_user_schema(self, project_id): self.gbq_connector, dataset, table, dict(fields=test_schema) ) - def test_upload_data_tokyo( - self, project_id, tokyo_dataset, bigquery_client - ): + def test_upload_data_tokyo(self, project_id, tokyo_dataset, bigquery_client): from google.cloud import bigquery test_size = 10 @@ -1544,8 +1478,7 @@ def test_upload_data_tokyo( table = bigquery_client.get_table( bigquery.TableReference( - bigquery.DatasetReference(project_id, tokyo_dataset), - "to_gbq_test", + bigquery.DatasetReference(project_id, tokyo_dataset), "to_gbq_test", ) ) assert table.num_rows > 0 @@ -1573,9 +1506,7 @@ def test_upload_data_tokyo_non_existing_dataset( table = bigquery_client.get_table( bigquery.TableReference( - bigquery.DatasetReference( - project_id, non_existing_tokyo_dataset - ), + bigquery.DatasetReference(project_id, non_existing_tokyo_dataset), "to_gbq_test", ) ) @@ -1585,15 +1516,11 @@ def test_upload_data_tokyo_non_existing_dataset( # _Dataset tests -def test_create_dataset( - bigquery_client, gbq_dataset, random_dataset_id, project_id -): +def test_create_dataset(bigquery_client, gbq_dataset, random_dataset_id, project_id): from google.cloud import bigquery gbq_dataset.create(random_dataset_id) - dataset_reference = bigquery.DatasetReference( - project_id, random_dataset_id - ) + dataset_reference = bigquery.DatasetReference(project_id, random_dataset_id) assert bigquery_client.get_dataset(dataset_reference) is not None @@ -1684,9 +1611,7 @@ def test_verify_schema_allows_flexible_column_order(gbq_table, gbq_connector): } gbq_table.create(table_id, test_schema_1) - assert verify_schema( - gbq_connector, gbq_table.dataset_id, table_id, test_schema_2 - ) + assert verify_schema(gbq_connector, gbq_table.dataset_id, table_id, test_schema_2) def test_verify_schema_fails_different_data_type(gbq_table, gbq_connector): @@ -1759,9 +1684,7 @@ def test_verify_schema_ignores_field_mode(gbq_table, gbq_connector): } gbq_table.create(table_id, test_schema_1) - assert verify_schema( - gbq_connector, gbq_table.dataset_id, table_id, test_schema_2 - ) + assert verify_schema(gbq_connector, gbq_table.dataset_id, table_id, test_schema_2) def test_retrieve_schema(gbq_table, gbq_connector): @@ -1769,24 +1692,9 @@ def test_retrieve_schema(gbq_table, gbq_connector): table_id = "test_retrieve_schema" test_schema = { "fields": [ - { - "name": "A", - "type": "FLOAT", - "mode": "NULLABLE", - "description": None, - }, - { - "name": "B", - "type": "FLOAT", - "mode": "NULLABLE", - "description": None, - }, - { - "name": "C", - "type": "STRING", - "mode": "NULLABLE", - "description": None, - }, + {"name": "A", "type": "FLOAT", "mode": "NULLABLE", "description": None}, + {"name": "B", "type": "FLOAT", "mode": "NULLABLE", "description": None}, + {"name": "C", "type": "STRING", "mode": "NULLABLE", "description": None}, { "name": "D", "type": "TIMESTAMP", @@ -1813,24 +1721,9 @@ def test_to_gbq_does_not_override_mode(gbq_table, gbq_connector): table_id = "test_to_gbq_does_not_override_mode" table_schema = { "fields": [ - { - "mode": "REQUIRED", - "name": "A", - "type": "FLOAT", - "description": "A", - }, - { - "mode": "NULLABLE", - "name": "B", - "type": "FLOAT", - "description": "B", - }, - { - "mode": "NULLABLE", - "name": "C", - "type": "STRING", - "description": "C", - }, + {"mode": "REQUIRED", "name": "A", "type": "FLOAT", "description": "A"}, + {"mode": "NULLABLE", "name": "B", "type": "FLOAT", "description": "B"}, + {"mode": "NULLABLE", "name": "C", "type": "STRING", "description": "C"}, ] } @@ -1842,6 +1735,4 @@ def test_to_gbq_does_not_override_mode(gbq_table, gbq_connector): if_exists="append", ) - assert verify_schema( - gbq_connector, gbq_table.dataset_id, table_id, table_schema - ) + assert verify_schema(gbq_connector, gbq_table.dataset_id, table_id, table_schema) diff --git a/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py b/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py index 8b9c7ecc31e3..8440948a3e73 100644 --- a/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py +++ b/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py @@ -34,10 +34,7 @@ def test_empty_results(method_under_test, query_string): See: https://github.com/pydata/pandas-gbq/issues/299 """ - df = method_under_test( - query_string, - use_bqstorage_api=True, - ) + df = method_under_test(query_string, use_bqstorage_api=True,) assert len(df.index) == 0 diff --git a/packages/pandas-gbq/tests/system/test_to_gbq.py b/packages/pandas-gbq/tests/system/test_to_gbq.py index 59435c3351ef..f500942184ff 100644 --- a/packages/pandas-gbq/tests/system/test_to_gbq.py +++ b/packages/pandas-gbq/tests/system/test_to_gbq.py @@ -19,9 +19,7 @@ def method_under_test(credentials): return functools.partial(pandas_gbq.to_gbq, credentials=credentials) -def test_float_round_trip( - method_under_test, random_dataset_id, bigquery_client -): +def test_float_round_trip(method_under_test, random_dataset_id, bigquery_client): """Ensure that 64-bit floating point numbers are unchanged. See: https://github.com/pydata/pandas-gbq/issues/326 @@ -47,7 +45,5 @@ def test_float_round_trip( round_trip = bigquery_client.list_rows(table_id).to_dataframe() round_trip_floats = round_trip["float_col"].sort_values() pandas.testing.assert_series_equal( - round_trip_floats, - input_floats, - check_exact=True, + round_trip_floats, input_floats, check_exact=True, ) diff --git a/packages/pandas-gbq/tests/unit/__init__.py b/packages/pandas-gbq/tests/unit/__init__.py index edbca6c3c489..c9ab850639a6 100644 --- a/packages/pandas-gbq/tests/unit/__init__.py +++ b/packages/pandas-gbq/tests/unit/__init__.py @@ -1,4 +1,3 @@ # Copyright (c) 2017 pandas-gbq Authors All rights reserved. # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. - diff --git a/packages/pandas-gbq/tests/unit/test_auth.py b/packages/pandas-gbq/tests/unit/test_auth.py index ca5447462c5c..c101942ed950 100644 --- a/packages/pandas-gbq/tests/unit/test_auth.py +++ b/packages/pandas-gbq/tests/unit/test_auth.py @@ -51,13 +51,9 @@ def mock_default_credentials(scopes=None, request=None): return (None, None) monkeypatch.setattr(google.auth, "default", mock_default_credentials) - mock_user_credentials = mock.create_autospec( - google.auth.credentials.Credentials - ) + mock_user_credentials = mock.create_autospec(google.auth.credentials.Credentials) - mock_cache = mock.create_autospec( - pydata_google_auth.cache.CredentialsCache - ) + mock_cache = mock.create_autospec(pydata_google_auth.cache.CredentialsCache) mock_cache.load.return_value = mock_user_credentials monkeypatch.setattr(auth, "get_credentials_cache", lambda _: mock_cache) @@ -71,6 +67,4 @@ def test_get_credentials_cache_w_reauth(): import pydata_google_auth.cache cache = auth.get_credentials_cache(True) - assert isinstance( - cache, pydata_google_auth.cache.WriteOnlyCredentialsCache - ) + assert isinstance(cache, pydata_google_auth.cache.WriteOnlyCredentialsCache) diff --git a/packages/pandas-gbq/tests/unit/test_features.py b/packages/pandas-gbq/tests/unit/test_features.py index d1d5af815b54..b10b0fa8afdf 100644 --- a/packages/pandas-gbq/tests/unit/test_features.py +++ b/packages/pandas-gbq/tests/unit/test_features.py @@ -23,9 +23,7 @@ def fresh_bigquery_version(monkeypatch): ("2.12.0", True), ], ) -def test_bigquery_has_from_dataframe_with_csv( - monkeypatch, bigquery_version, expected -): +def test_bigquery_has_from_dataframe_with_csv(monkeypatch, bigquery_version, expected): import google.cloud.bigquery monkeypatch.setattr(google.cloud.bigquery, "__version__", bigquery_version) diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index 7476db3f37e2..3b6034128604 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -17,9 +17,7 @@ from pandas_gbq.features import FEATURES -pytestmark = pytest.mark.filterwarnings( - "ignore:credentials from Google Cloud SDK" -) +pytestmark = pytest.mark.filterwarnings("ignore:credentials from Google Cloud SDK") def _make_connector(project_id="some-project", **kwargs): @@ -29,18 +27,14 @@ def _make_connector(project_id="some-project", **kwargs): def mock_get_credentials_no_project(*args, **kwargs): import google.auth.credentials - mock_credentials = mock.create_autospec( - google.auth.credentials.Credentials - ) + mock_credentials = mock.create_autospec(google.auth.credentials.Credentials) return mock_credentials, None def mock_get_credentials(*args, **kwargs): import google.auth.credentials - mock_credentials = mock.create_autospec( - google.auth.credentials.Credentials - ) + mock_credentials = mock.create_autospec(google.auth.credentials.Credentials) return mock_credentials, "default-project" @@ -48,9 +42,7 @@ def mock_get_credentials(*args, **kwargs): def mock_service_account_credentials(): import google.oauth2.service_account - mock_credentials = mock.create_autospec( - google.oauth2.service_account.Credentials - ) + mock_credentials = mock.create_autospec(google.oauth2.service_account.Credentials) return mock_credentials @@ -58,9 +50,7 @@ def mock_service_account_credentials(): def mock_compute_engine_credentials(): import google.auth.compute_engine - mock_credentials = mock.create_autospec( - google.auth.compute_engine.Credentials - ) + mock_credentials = mock.create_autospec(google.auth.compute_engine.Credentials) return mock_credentials @@ -104,9 +94,7 @@ def test_GbqConnector_get_client_w_old_bq(monkeypatch, mock_bigquery_client): connector.get_client() # No client_info argument. - mock_bigquery_client.assert_called_with( - credentials=mock.ANY, project=mock.ANY - ) + mock_bigquery_client.assert_called_with(credentials=mock.ANY, project=mock.ANY) def test_GbqConnector_get_client_w_new_bq(mock_bigquery_client): @@ -119,9 +107,7 @@ def test_GbqConnector_get_client_w_new_bq(mock_bigquery_client): connector.get_client() _, kwargs = mock_bigquery_client.call_args - assert kwargs["client_info"].user_agent == "pandas-{}".format( - pandas.__version__ - ) + assert kwargs["client_info"].user_agent == "pandas-{}".format(pandas.__version__) def test_to_gbq_should_fail_if_invalid_table_name_passed(): @@ -132,18 +118,14 @@ def test_to_gbq_should_fail_if_invalid_table_name_passed(): def test_to_gbq_with_no_project_id_given_should_fail(monkeypatch): import pydata_google_auth - monkeypatch.setattr( - pydata_google_auth, "default", mock_get_credentials_no_project - ) + monkeypatch.setattr(pydata_google_auth, "default", mock_get_credentials_no_project) with pytest.raises(ValueError, match="Could not determine project ID"): gbq.to_gbq(DataFrame([[1]]), "dataset.tablename") @pytest.mark.parametrize(["verbose"], [(True,), (False,)]) -def test_to_gbq_with_verbose_new_pandas_warns_deprecation( - monkeypatch, verbose -): +def test_to_gbq_with_verbose_new_pandas_warns_deprecation(monkeypatch, verbose): monkeypatch.setattr( type(FEATURES), "pandas_has_deprecated_verbose", @@ -168,9 +150,7 @@ def test_to_gbq_wo_verbose_w_new_pandas_no_warnings(monkeypatch, recwarn): mock.PropertyMock(return_value=True), ) try: - gbq.to_gbq( - DataFrame([[1]]), "dataset.tablename", project_id="my-project" - ) + gbq.to_gbq(DataFrame([[1]]), "dataset.tablename", project_id="my-project") except gbq.TableCreationError: pass assert len(recwarn) == 0 @@ -206,9 +186,7 @@ def test_to_gbq_with_private_key_raises_notimplementederror(): def test_to_gbq_doesnt_run_query(mock_bigquery_client): try: - gbq.to_gbq( - DataFrame([[1]]), "dataset.tablename", project_id="my-project" - ) + gbq.to_gbq(DataFrame([[1]]), "dataset.tablename", project_id="my-project") except gbq.TableCreationError: pass @@ -218,8 +196,8 @@ def test_to_gbq_doesnt_run_query(mock_bigquery_client): def test_to_gbq_w_empty_df(mock_bigquery_client): import google.api_core.exceptions - mock_bigquery_client.get_table.side_effect = ( - google.api_core.exceptions.NotFound("my_table") + mock_bigquery_client.get_table.side_effect = google.api_core.exceptions.NotFound( + "my_table" ) gbq.to_gbq(DataFrame(), "my_dataset.my_table", project_id="1234") mock_bigquery_client.create_table.assert_called_with(mock.ANY) @@ -234,8 +212,8 @@ def test_to_gbq_w_default_project(mock_bigquery_client): import google.api_core.exceptions from google.cloud.bigquery.table import TableReference - mock_bigquery_client.get_table.side_effect = ( - google.api_core.exceptions.NotFound("my_table") + mock_bigquery_client.get_table.side_effect = google.api_core.exceptions.NotFound( + "my_table" ) gbq.to_gbq(DataFrame(), "my_dataset.my_table") @@ -254,13 +232,11 @@ def test_to_gbq_w_project_table(mock_bigquery_client): import google.api_core.exceptions from google.cloud.bigquery.table import TableReference - mock_bigquery_client.get_table.side_effect = ( - google.api_core.exceptions.NotFound("my_table") + mock_bigquery_client.get_table.side_effect = google.api_core.exceptions.NotFound( + "my_table" ) gbq.to_gbq( - DataFrame(), - "project_table.my_dataset.my_table", - project_id="project_client", + DataFrame(), "project_table.my_dataset.my_table", project_id="project_client", ) mock_bigquery_client.get_table.assert_called_with( @@ -274,11 +250,11 @@ def test_to_gbq_w_project_table(mock_bigquery_client): def test_to_gbq_creates_dataset(mock_bigquery_client): import google.api_core.exceptions - mock_bigquery_client.get_table.side_effect = ( - google.api_core.exceptions.NotFound("my_table") + mock_bigquery_client.get_table.side_effect = google.api_core.exceptions.NotFound( + "my_table" ) - mock_bigquery_client.get_dataset.side_effect = ( - google.api_core.exceptions.NotFound("my_dataset") + mock_bigquery_client.get_dataset.side_effect = google.api_core.exceptions.NotFound( + "my_dataset" ) gbq.to_gbq(DataFrame([[1]]), "my_dataset.my_table", project_id="1234") mock_bigquery_client.create_dataset.assert_called_with(mock.ANY) @@ -287,9 +263,7 @@ def test_to_gbq_creates_dataset(mock_bigquery_client): def test_read_gbq_with_no_project_id_given_should_fail(monkeypatch): import pydata_google_auth - monkeypatch.setattr( - pydata_google_auth, "default", mock_get_credentials_no_project - ) + monkeypatch.setattr(pydata_google_auth, "default", mock_get_credentials_no_project) with pytest.raises(ValueError, match="Could not determine project ID"): gbq.read_gbq("SELECT 1", dialect="standard") @@ -305,9 +279,7 @@ def test_read_gbq_with_inferred_project_id_from_service_account_credentials( ): mock_service_account_credentials.project_id = "service_account_project_id" df = gbq.read_gbq( - "SELECT 1", - dialect="standard", - credentials=mock_service_account_credentials, + "SELECT 1", dialect="standard", credentials=mock_service_account_credentials, ) assert df is not None mock_bigquery_client.query.assert_called_once_with( @@ -323,9 +295,7 @@ def test_read_gbq_without_inferred_project_id_from_compute_engine_credentials( ): with pytest.raises(ValueError, match="Could not determine project ID"): gbq.read_gbq( - "SELECT 1", - dialect="standard", - credentials=mock_compute_engine_credentials, + "SELECT 1", dialect="standard", credentials=mock_compute_engine_credentials, ) @@ -341,9 +311,7 @@ def test_read_gbq_with_max_results_ten(monkeypatch, mock_bigquery_client): @pytest.mark.parametrize(["verbose"], [(True,), (False,)]) -def test_read_gbq_with_verbose_new_pandas_warns_deprecation( - monkeypatch, verbose -): +def test_read_gbq_with_verbose_new_pandas_warns_deprecation(monkeypatch, verbose): monkeypatch.setattr( type(FEATURES), "pandas_has_deprecated_verbose", @@ -370,8 +338,7 @@ def test_read_gbq_with_old_bq_raises_importerror(monkeypatch): monkeypatch.setattr(FEATURES, "_bigquery_installed_version", None) with pytest.raises(ImportError, match="google-cloud-bigquery"): gbq.read_gbq( - "SELECT 1", - project_id="my-project", + "SELECT 1", project_id="my-project", ) @@ -382,10 +349,7 @@ def test_read_gbq_with_verbose_old_pandas_no_warnings(monkeypatch, recwarn): mock.PropertyMock(return_value=False), ) gbq.read_gbq( - "SELECT 1", - project_id="my-project", - dialect="standard", - verbose=True, + "SELECT 1", project_id="my-project", dialect="standard", verbose=True, ) assert len(recwarn) == 0 @@ -411,9 +375,7 @@ def test_read_gbq_with_configuration_duplicate_query_raises_error(): with pytest.raises( ValueError, match="Query statement can't be specified inside config" ): - gbq.read_gbq( - "SELECT 1", configuration={"query": {"query": "SELECT 2"}} - ) + gbq.read_gbq("SELECT 1", configuration={"query": {"query": "SELECT 2"}}) def test_generate_bq_schema_deprecated(): @@ -469,9 +431,7 @@ def test_load_does_not_modify_schema_arg(mock_bigquery_client): assert original_schema == original_schema_cp -def test_read_gbq_passes_dtypes( - mock_bigquery_client, mock_service_account_credentials -): +def test_read_gbq_passes_dtypes(mock_bigquery_client, mock_service_account_credentials): mock_service_account_credentials.project_id = "service_account_project_id" df = gbq.read_gbq( "SELECT 1 AS int_col", @@ -504,15 +464,11 @@ def test_read_gbq_use_bqstorage_api( mock_list_rows = mock_bigquery_client.list_rows("dest", max_results=100) mock_list_rows.to_dataframe.assert_called_once_with( - create_bqstorage_client=True, - dtypes=mock.ANY, - progress_bar_type=mock.ANY, + create_bqstorage_client=True, dtypes=mock.ANY, progress_bar_type=mock.ANY, ) -def test_read_gbq_calls_tqdm( - mock_bigquery_client, mock_service_account_credentials -): +def test_read_gbq_calls_tqdm(mock_bigquery_client, mock_service_account_credentials): mock_service_account_credentials.project_id = "service_account_project_id" df = gbq.read_gbq( "SELECT 1", diff --git a/packages/pandas-gbq/tests/unit/test_load.py b/packages/pandas-gbq/tests/unit/test_load.py index 353b8bd122b8..d00495a6548e 100644 --- a/packages/pandas-gbq/tests/unit/test_load.py +++ b/packages/pandas-gbq/tests/unit/test_load.py @@ -56,13 +56,9 @@ def test_encode_chunk_with_floats(): StringIO(input_csv), header=None, float_precision="round_trip" ) csv_buffer = load.encode_chunk(input_df) - round_trip = pandas.read_csv( - csv_buffer, header=None, float_precision="round_trip" - ) + round_trip = pandas.read_csv(csv_buffer, header=None, float_precision="round_trip") pandas.testing.assert_frame_equal( - round_trip, - input_df, - check_exact=True, + round_trip, input_df, check_exact=True, ) @@ -95,9 +91,7 @@ def test_encode_chunks_with_chunksize_none(): assert len(chunk.index) == 6 -@pytest.mark.parametrize( - ["bigquery_has_from_dataframe_with_csv"], [(True,), (False,)] -) +@pytest.mark.parametrize(["bigquery_has_from_dataframe_with_csv"], [(True,), (False,)]) def test_load_chunks_omits_policy_tags( monkeypatch, mock_bigquery_client, bigquery_has_from_dataframe_with_csv ): @@ -118,14 +112,10 @@ def test_load_chunks_omits_policy_tags( "my-project.my_dataset.my_table" ) schema = { - "fields": [ - {"name": "col1", "type": "INT64", "policyTags": ["tag1", "tag2"]} - ] + "fields": [{"name": "col1", "type": "INT64", "policyTags": ["tag1", "tag2"]}] } - _ = list( - load.load_chunks(mock_bigquery_client, df, destination, schema=schema) - ) + _ = list(load.load_chunks(mock_bigquery_client, df, destination, schema=schema)) mock_load = load_method(mock_bigquery_client) assert mock_load.called diff --git a/packages/pandas-gbq/tests/unit/test_schema.py b/packages/pandas-gbq/tests/unit/test_schema.py index bd04508e1f7e..743ddc26111c 100644 --- a/packages/pandas-gbq/tests/unit/test_schema.py +++ b/packages/pandas-gbq/tests/unit/test_schema.py @@ -24,21 +24,12 @@ def module_under_test(): {"name": "B", "type": "FLOAT64"}, {"name": "C", "type": "STRING"}, ], - [ - {"name": "A", "type": "FLOAT64"}, - {"name": "B", "type": "FLOAT"}, - ], + [{"name": "A", "type": "FLOAT64"}, {"name": "B", "type": "FLOAT"}], ), # Original schema from API may contain legacy SQL datatype names. # https://github.com/pydata/pandas-gbq/issues/322 - ( - [{"name": "A", "type": "INTEGER"}], - [{"name": "A", "type": "INT64"}], - ), - ( - [{"name": "A", "type": "BOOL"}], - [{"name": "A", "type": "BOOLEAN"}], - ), + ([{"name": "A", "type": "INTEGER"}], [{"name": "A", "type": "INT64"}],), + ([{"name": "A", "type": "BOOL"}], [{"name": "A", "type": "BOOLEAN"}],), ( # TODO: include sub-fields when struct uploads are supported. [{"name": "A", "type": "STRUCT"}], @@ -65,10 +56,7 @@ def test_schema_is_subset_fails_if_not_subset(module_under_test): ] } tested_schema = { - "fields": [ - {"name": "A", "type": "FLOAT"}, - {"name": "C", "type": "FLOAT"}, - ] + "fields": [{"name": "A", "type": "FLOAT"}, {"name": "C", "type": "FLOAT"}] } assert not module_under_test.schema_is_subset(table_schema, tested_schema) @@ -160,8 +148,6 @@ def test_generate_bq_schema(module_under_test, dataframe, expected_schema): ), ], ) -def test_update_schema( - module_under_test, schema_old, schema_new, expected_output -): +def test_update_schema(module_under_test, schema_old, schema_new, expected_output): output = module_under_test.update_schema(schema_old, schema_new) assert output == expected_output diff --git a/packages/pandas-gbq/tests/unit/test_timestamp.py b/packages/pandas-gbq/tests/unit/test_timestamp.py index 6c9e32823a6b..406643d07e6b 100644 --- a/packages/pandas-gbq/tests/unit/test_timestamp.py +++ b/packages/pandas-gbq/tests/unit/test_timestamp.py @@ -33,9 +33,7 @@ def test_localize_df_with_empty_dataframe(module_under_test): def test_localize_df_with_no_timestamp_columns(module_under_test): - df = pandas.DataFrame( - {"integer_col": [1, 2, 3], "float_col": [0.1, 0.2, 0.3]} - ) + df = pandas.DataFrame({"integer_col": [1, 2, 3], "float_col": [0.1, 0.2, 0.3]}) original = df.copy() bq_schema = [ {"name": "integer_col", "type": "INTEGER"}, @@ -54,11 +52,7 @@ def test_localize_df_with_timestamp_column(module_under_test): { "integer_col": [1, 2, 3], "timestamp_col": pandas.Series( - [ - "2011-01-01 01:02:03", - "2012-02-02 04:05:06", - "2013-03-03 07:08:09", - ], + ["2011-01-01 01:02:03", "2012-02-02 04:05:06", "2013-03-03 07:08:09"], dtype="datetime64[ns]", ), "float_col": [0.1, 0.2, 0.3], From 2865501a82e208183f4da65faf74e7324aef6dc4 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 24 Sep 2021 12:22:23 -0500 Subject: [PATCH 231/519] doc: unify layout with Google client libraries (#389) * doc: unify layout with Google client libraries * migrate samples to new layout too --- .../pandas-gbq/docs/{source => }/Makefile | 0 packages/pandas-gbq/docs/README.rst | 10 - .../docs/{source => }/_static/style.css | 0 .../docs/{source => }/_templates/layout.html | 0 packages/pandas-gbq/docs/{source => }/api.rst | 0 packages/pandas-gbq/docs/changelog.md | 1 + packages/pandas-gbq/docs/{source => }/conf.py | 241 ++++++++---------- .../docs/{source => }/contributing.rst | 2 +- .../{source => }/howto/authentication.rst | 0 .../pandas-gbq/docs/{source => }/index.rst | 0 .../pandas-gbq/docs/{source => }/install.rst | 0 .../pandas-gbq/docs/{source => }/intro.rst | 6 +- .../pandas-gbq/docs/{source => }/privacy.rst | 0 .../pandas-gbq/docs/{source => }/reading.rst | 4 +- packages/pandas-gbq/docs/samples | 1 + packages/pandas-gbq/docs/source/changelog.md | 1 - packages/pandas-gbq/docs/source/samples | 1 - .../pandas-gbq/docs/{source => }/writing.rst | 2 +- packages/pandas-gbq/noxfile.py | 14 +- packages/pandas-gbq/samples/samples | 1 - .../samples/{ => snippets}/__init__.py | 0 .../samples/{tests => snippets}/conftest.py | 0 .../samples/{ => snippets}/read_gbq_legacy.py | 0 .../samples/{ => snippets}/read_gbq_simple.py | 0 .../{tests => snippets}/test_read_gbq.py | 4 +- .../{tests => snippets}/test_to_gbq.py | 2 +- .../samples/{ => snippets}/to_gbq_simple.py | 0 packages/pandas-gbq/samples/tests/__init__.py | 3 - 28 files changed, 129 insertions(+), 164 deletions(-) rename packages/pandas-gbq/docs/{source => }/Makefile (100%) delete mode 100644 packages/pandas-gbq/docs/README.rst rename packages/pandas-gbq/docs/{source => }/_static/style.css (100%) rename packages/pandas-gbq/docs/{source => }/_templates/layout.html (100%) rename packages/pandas-gbq/docs/{source => }/api.rst (100%) create mode 120000 packages/pandas-gbq/docs/changelog.md rename packages/pandas-gbq/docs/{source => }/conf.py (67%) rename packages/pandas-gbq/docs/{source => }/contributing.rst (99%) rename packages/pandas-gbq/docs/{source => }/howto/authentication.rst (100%) rename packages/pandas-gbq/docs/{source => }/index.rst (100%) rename packages/pandas-gbq/docs/{source => }/install.rst (100%) rename packages/pandas-gbq/docs/{source => }/intro.rst (94%) rename packages/pandas-gbq/docs/{source => }/privacy.rst (100%) rename packages/pandas-gbq/docs/{source => }/reading.rst (97%) create mode 120000 packages/pandas-gbq/docs/samples delete mode 120000 packages/pandas-gbq/docs/source/changelog.md delete mode 120000 packages/pandas-gbq/docs/source/samples rename packages/pandas-gbq/docs/{source => }/writing.rst (97%) delete mode 120000 packages/pandas-gbq/samples/samples rename packages/pandas-gbq/samples/{ => snippets}/__init__.py (100%) rename packages/pandas-gbq/samples/{tests => snippets}/conftest.py (100%) rename packages/pandas-gbq/samples/{ => snippets}/read_gbq_legacy.py (100%) rename packages/pandas-gbq/samples/{ => snippets}/read_gbq_simple.py (100%) rename packages/pandas-gbq/samples/{tests => snippets}/test_read_gbq.py (90%) rename packages/pandas-gbq/samples/{tests => snippets}/test_to_gbq.py (93%) rename packages/pandas-gbq/samples/{ => snippets}/to_gbq_simple.py (100%) delete mode 100644 packages/pandas-gbq/samples/tests/__init__.py diff --git a/packages/pandas-gbq/docs/source/Makefile b/packages/pandas-gbq/docs/Makefile similarity index 100% rename from packages/pandas-gbq/docs/source/Makefile rename to packages/pandas-gbq/docs/Makefile diff --git a/packages/pandas-gbq/docs/README.rst b/packages/pandas-gbq/docs/README.rst deleted file mode 100644 index 8cd89d11da58..000000000000 --- a/packages/pandas-gbq/docs/README.rst +++ /dev/null @@ -1,10 +0,0 @@ -To build a local copy of the pandas-gbq docs, install the programs in -requirements-docs.txt and run 'make html'. If you use the conda package manager -these commands suffice:: - - git clone git@github.com:pydata/pandas-gbq.git - cd dask/docs - conda create -n pandas-gbq-docs --file requirements-docs.txt - source activate pandas-gbq-docs - make html - open build/html/index.html diff --git a/packages/pandas-gbq/docs/source/_static/style.css b/packages/pandas-gbq/docs/_static/style.css similarity index 100% rename from packages/pandas-gbq/docs/source/_static/style.css rename to packages/pandas-gbq/docs/_static/style.css diff --git a/packages/pandas-gbq/docs/source/_templates/layout.html b/packages/pandas-gbq/docs/_templates/layout.html similarity index 100% rename from packages/pandas-gbq/docs/source/_templates/layout.html rename to packages/pandas-gbq/docs/_templates/layout.html diff --git a/packages/pandas-gbq/docs/source/api.rst b/packages/pandas-gbq/docs/api.rst similarity index 100% rename from packages/pandas-gbq/docs/source/api.rst rename to packages/pandas-gbq/docs/api.rst diff --git a/packages/pandas-gbq/docs/changelog.md b/packages/pandas-gbq/docs/changelog.md new file mode 120000 index 000000000000..04c99a55caae --- /dev/null +++ b/packages/pandas-gbq/docs/changelog.md @@ -0,0 +1 @@ +../CHANGELOG.md \ No newline at end of file diff --git a/packages/pandas-gbq/docs/source/conf.py b/packages/pandas-gbq/docs/conf.py similarity index 67% rename from packages/pandas-gbq/docs/source/conf.py rename to packages/pandas-gbq/docs/conf.py index b250e7d088d0..a4eed21b11c6 100644 --- a/packages/pandas-gbq/docs/source/conf.py +++ b/packages/pandas-gbq/docs/conf.py @@ -1,11 +1,19 @@ -# Copyright (c) 2017 pandas-gbq Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - # -*- coding: utf-8 -*- +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 # -# pandas-gbq documentation build configuration file, created by -# sphinx-quickstart on Wed Feb 8 10:52:12 2017. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# pandas-gbq documentation build configuration file # # This file is execfile()d with the current directory set to its # containing dir. @@ -16,23 +24,25 @@ # All configuration values have a default; values that are commented out # serve to show the default. +import sys +import os +import shlex + # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -# -import datetime -import os -import sys +sys.path.insert(0, os.path.abspath("..")) -import pandas_gbq +# For plugins that can not read conf.py. +# See also: https://github.com/docascode/sphinx-docfx-yaml/issues/85 +sys.path.insert(0, os.path.abspath(".")) -# sys.path.insert(0, os.path.abspath('.')) +__version__ = "" # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -# -# needs_sphinx = '1.0' +needs_sphinx = "1.5.5" # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom @@ -40,47 +50,48 @@ extensions = [ "sphinx.ext.autodoc", "sphinx.ext.autosummary", - "sphinx.ext.doctest", - "sphinx.ext.extlinks", - "sphinx.ext.todo", - "numpydoc", # used to parse numpy-style docstrings for autodoc - "IPython.sphinxext.ipython_console_highlighting", - "IPython.sphinxext.ipython_directive", "sphinx.ext.intersphinx", "sphinx.ext.coverage", - "sphinx.ext.ifconfig", + "sphinx.ext.doctest", + "sphinx.ext.napoleon", + "sphinx.ext.todo", + "sphinx.ext.viewcode", "recommonmark", ] +# autodoc/autosummary flags +autoclass_content = "both" +autodoc_default_options = {"members": True} +autosummary_generate = True + + # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: -# # source_suffix = ['.rst', '.md'] source_suffix = [".rst", ".md"] # The encoding of source files. -# # source_encoding = 'utf-8-sig' -# The master toctree document. -master_doc = "index" +# The root toctree document. +root_doc = "index" # General information about the project. -project = u"pandas-gbq" -copyright = u"2017-{}, PyData Development Team".format(datetime.datetime.now().year) -author = u"PyData Development Team" +project = "pandas-gbq" +copyright = "2019, Google" +author = "Google APIs" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # -# The short X.Y version. -version = pandas_gbq.__version__ # The full version, including alpha/beta/rc tags. -release = version +release = __version__ +# The short X.Y version. +version = ".".join(release.split(".")[0:2]) # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -91,11 +102,8 @@ # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -# # today = '' -# # Else, today_fmt is used as the format for a strftime call. -# # today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and @@ -110,21 +118,17 @@ # The reST default role (used for this markup: `text`) to use for all # documents. -# # default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -# # add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -# # add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -# # show_authors = False # The name of the Pygments (syntax highlighting) style to use. @@ -137,53 +141,45 @@ # keep_warnings = False # If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = False +todo_include_todos = True # -- Options for HTML output ---------------------------------------------- -# Taken from docs.readthedocs.io: -# on_rtd is whether we are on readthedocs.io -on_rtd = os.environ.get("READTHEDOCS", None) == "True" - -if not on_rtd: # only import and set the theme if we're building docs locally - import sphinx_rtd_theme - - html_theme = "sphinx_rtd_theme" - html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] - # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -# -# html_theme = 'alabaster' +html_theme = "alabaster" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -# -# html_theme_options = {} +html_theme_options = { + "description": "Google Cloud Client Libraries for pandas-gbq", + "github_user": "googleapis", + "github_repo": "python-bigquery-pandas", + "github_banner": True, + "font_family": "'Roboto', Georgia, sans", + "head_font_family": "'Roboto', Georgia, serif", + "code_font_family": "'Roboto Mono', 'Consolas', monospace", +} # Add any paths that contain custom themes here, relative to this directory. # html_theme_path = [] -# The name for this set of Sphinx documents. -# " v documentation" by default. -# -# html_title = u'pandas-gbq v0.1.0' +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -# # html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -# # html_logo = None -# The name of an image file (relative to this directory) to use as a favicon of -# the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -# # html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, @@ -194,57 +190,44 @@ # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. -# # html_extra_path = [] -# If not None, a 'Last updated on:' timestamp is inserted at every page -# bottom, using the given strftime format. -# The empty string is equivalent to '%b %d, %Y'. -# -# html_last_updated_fmt = None +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +# html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -# # html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -# # html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -# # html_additional_pages = {} # If false, no module index is generated. -# # html_domain_indices = True # If false, no index is generated. -# # html_use_index = True # If true, the index is split into individual pages for each letter. -# # html_split_index = False # If true, links to the reST sources are added to the pages. -# # html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -# # html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -# # html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -# # html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). @@ -253,84 +236,70 @@ # Language to be used for generating the HTML full-text search index. # Sphinx supports the following languages: # 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' -# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr', 'zh' -# +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' # html_search_language = 'en' # A dictionary with options for the search language support, empty by default. -# 'ja' uses this config value. -# 'zh' user can custom change `jieba` dictionary path. -# +# Now only 'ja' uses this config value # html_search_options = {'type': 'default'} # The name of a javascript file (relative to the configuration directory) that # implements a search results scorer. If empty, the default will be used. -# # html_search_scorer = 'scorer.js' # Output file base name for HTML help builder. -htmlhelp_basename = "pandas-gbqdoc" +htmlhelp_basename = "pandas-gbq-doc" + +# -- Options for warnings ------------------------------------------------------ + + +suppress_warnings = [ + # Temporarily suppress this to avoid "more than one target found for + # cross-reference" warning, which are intractable for us to avoid while in + # a mono-repo. + # See https://github.com/sphinx-doc/sphinx/blob + # /2a65ffeef5c107c19084fabdd706cdff3f52d93c/sphinx/domains/python.py#L843 + "ref.python" +] # -- Options for LaTeX output --------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', + #'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', + #'pointsize': '10pt', # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', + #'preamble': '', # Latex figure (float) alignment - # - # 'figure_align': 'htbp', + #'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - ( - master_doc, - "pandas-gbq.tex", - u"pandas-gbq Documentation", - u"PyData Development Team", - "manual", - ) + (root_doc, "pandas-gbq.tex", "pandas-gbq Documentation", author, "manual",) ] # The name of an image file (relative to this directory) to place at the top of # the title page. -# # latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -# # latex_use_parts = False # If true, show page references after internal links. -# # latex_show_pagerefs = False # If true, show URL addresses after external links. -# # latex_show_urls = False # Documents to append as an appendix to all manuals. -# # latex_appendices = [] -# It false, will not define \strong, \code, itleref, \crossref ... but only -# \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added -# packages. -# -# latex_keep_old_macro_names = True - # If false, no module index is generated. -# # latex_domain_indices = True @@ -338,10 +307,9 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [(master_doc, "pandas-gbq", u"pandas-gbq Documentation", [author], 1)] +man_pages = [(root_doc, "pandas-gbq", "pandas-gbq Documentation", [author], 1,)] # If true, show URL addresses after external links. -# # man_show_urls = False @@ -352,42 +320,53 @@ # dir menu entry, description, category) texinfo_documents = [ ( - master_doc, + root_doc, "pandas-gbq", - u"pandas-gbq Documentation", + "pandas-gbq Documentation", author, "pandas-gbq", - "One line description of project.", - "Miscellaneous", + "pandas-gbq Library", + "APIs", ) ] # Documents to append as an appendix to all manuals. -# # texinfo_appendices = [] # If false, no module index is generated. -# # texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -# # texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. -# # texinfo_no_detailmenu = False -# Configuration for intersphinx: +# Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = { - "https://docs.python.org/": None, - "https://pandas.pydata.org/pandas-docs/stable/": None, - "https://pydata-google-auth.readthedocs.io/en/latest/": None, - "https://google-auth.readthedocs.io/en/latest/": None, + "python": ("https://python.readthedocs.org/en/latest/", None), + "google-auth": ("https://googleapis.dev/python/google-auth/latest/", None), + "google.api_core": ("https://googleapis.dev/python/google-api-core/latest/", None,), + "grpc": ("https://grpc.github.io/grpc/python/", None), + "proto-plus": ("https://proto-plus-python.readthedocs.io/en/latest/", None), + "protobuf": ("https://googleapis.dev/python/protobuf/latest/", None), + "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None), + "pydata-google-auth": ( + "https://pydata-google-auth.readthedocs.io/en/latest/", + None, + ), } -extlinks = { - "issue": ("https://github.com/pydata/pandas-gbq/issues/%s", "GH#"), - "pr": ("https://github.com/pydata/pandas-gbq/pull/%s", "GH#"), -} + +# Napoleon settings +napoleon_google_docstring = True +napoleon_numpy_docstring = True +napoleon_include_private_with_doc = False +napoleon_include_special_with_doc = True +napoleon_use_admonition_for_examples = False +napoleon_use_admonition_for_notes = False +napoleon_use_admonition_for_references = False +napoleon_use_ivar = False +napoleon_use_param = True +napoleon_use_rtype = True diff --git a/packages/pandas-gbq/docs/source/contributing.rst b/packages/pandas-gbq/docs/contributing.rst similarity index 99% rename from packages/pandas-gbq/docs/source/contributing.rst rename to packages/pandas-gbq/docs/contributing.rst index 3bd868495940..4a17e8033b89 100644 --- a/packages/pandas-gbq/docs/source/contributing.rst +++ b/packages/pandas-gbq/docs/contributing.rst @@ -335,7 +335,7 @@ Documenting your code --------------------- Changes should follow convential commits. The release-please bot uses the -commit message to create an ongoing change log. +commit message to create an ongoing change log. If your code is an enhancement, it is most likely necessary to add usage examples to the existing documentation. Further, to let users know when diff --git a/packages/pandas-gbq/docs/source/howto/authentication.rst b/packages/pandas-gbq/docs/howto/authentication.rst similarity index 100% rename from packages/pandas-gbq/docs/source/howto/authentication.rst rename to packages/pandas-gbq/docs/howto/authentication.rst diff --git a/packages/pandas-gbq/docs/source/index.rst b/packages/pandas-gbq/docs/index.rst similarity index 100% rename from packages/pandas-gbq/docs/source/index.rst rename to packages/pandas-gbq/docs/index.rst diff --git a/packages/pandas-gbq/docs/source/install.rst b/packages/pandas-gbq/docs/install.rst similarity index 100% rename from packages/pandas-gbq/docs/source/install.rst rename to packages/pandas-gbq/docs/install.rst diff --git a/packages/pandas-gbq/docs/source/intro.rst b/packages/pandas-gbq/docs/intro.rst similarity index 94% rename from packages/pandas-gbq/docs/source/intro.rst rename to packages/pandas-gbq/docs/intro.rst index e506ebe78b56..c6774b15ec4c 100644 --- a/packages/pandas-gbq/docs/source/intro.rst +++ b/packages/pandas-gbq/docs/intro.rst @@ -27,7 +27,7 @@ Reading data from BigQuery Use the :func:`pandas_gbq.read_gbq` function to run a BigQuery query and download the results as a :class:`pandas.DataFrame` object. -.. literalinclude:: samples/read_gbq_simple.py +.. literalinclude:: samples/snippets/read_gbq_simple.py :language: python :dedent: 4 :start-after: [START bigquery_pandas_gbq_read_gbq_simple] @@ -44,7 +44,7 @@ longer queries. IPython & Jupyter by default attach a handler to the logger. If you're running in another process and want to see logs, or you want to see more verbose logs, you can do something like: -.. code-block:: ipython +.. code-block:: python import logging logger = logging.getLogger('pandas_gbq') @@ -57,7 +57,7 @@ Writing data to BigQuery Use the :func:`pandas_gbq.to_gbq` function to write a :class:`pandas.DataFrame` object to a BigQuery table. -.. literalinclude:: samples/to_gbq_simple.py +.. literalinclude:: samples/snippets/to_gbq_simple.py :language: python :dedent: 4 :start-after: [START bigquery_pandas_gbq_to_gbq_simple] diff --git a/packages/pandas-gbq/docs/source/privacy.rst b/packages/pandas-gbq/docs/privacy.rst similarity index 100% rename from packages/pandas-gbq/docs/source/privacy.rst rename to packages/pandas-gbq/docs/privacy.rst diff --git a/packages/pandas-gbq/docs/source/reading.rst b/packages/pandas-gbq/docs/reading.rst similarity index 97% rename from packages/pandas-gbq/docs/source/reading.rst rename to packages/pandas-gbq/docs/reading.rst index 67919c57f253..aaecf9a0f188 100644 --- a/packages/pandas-gbq/docs/source/reading.rst +++ b/packages/pandas-gbq/docs/reading.rst @@ -6,7 +6,7 @@ Reading Tables Use the :func:`pandas_gbq.read_gbq` function to run a BigQuery query and download the results as a :class:`pandas.DataFrame` object. -.. literalinclude:: samples/read_gbq_simple.py +.. literalinclude:: samples/snippets/read_gbq_simple.py :language: python :dedent: 4 :start-after: [START bigquery_pandas_gbq_read_gbq_simple] @@ -37,7 +37,7 @@ The ``dialect`` argument can be used to indicate whether to use BigQuery's ``'legacy'`` SQL or BigQuery's ``'standard'`` SQL. The default value is ``'standard'``. -.. literalinclude:: samples/read_gbq_legacy.py +.. literalinclude:: samples/snippets/read_gbq_legacy.py :language: python :dedent: 4 :start-after: [START bigquery_pandas_gbq_read_gbq_legacy] diff --git a/packages/pandas-gbq/docs/samples b/packages/pandas-gbq/docs/samples new file mode 120000 index 000000000000..e804737ed3a9 --- /dev/null +++ b/packages/pandas-gbq/docs/samples @@ -0,0 +1 @@ +../samples \ No newline at end of file diff --git a/packages/pandas-gbq/docs/source/changelog.md b/packages/pandas-gbq/docs/source/changelog.md deleted file mode 120000 index 699cc9e7b7c5..000000000000 --- a/packages/pandas-gbq/docs/source/changelog.md +++ /dev/null @@ -1 +0,0 @@ -../../CHANGELOG.md \ No newline at end of file diff --git a/packages/pandas-gbq/docs/source/samples b/packages/pandas-gbq/docs/source/samples deleted file mode 120000 index 47920198b9bd..000000000000 --- a/packages/pandas-gbq/docs/source/samples +++ /dev/null @@ -1 +0,0 @@ -../../samples \ No newline at end of file diff --git a/packages/pandas-gbq/docs/source/writing.rst b/packages/pandas-gbq/docs/writing.rst similarity index 97% rename from packages/pandas-gbq/docs/source/writing.rst rename to packages/pandas-gbq/docs/writing.rst index a6a5c1238320..6c1be27220a8 100644 --- a/packages/pandas-gbq/docs/source/writing.rst +++ b/packages/pandas-gbq/docs/writing.rst @@ -6,7 +6,7 @@ Writing Tables Use the :func:`pandas_gbq.to_gbq` function to write a :class:`pandas.DataFrame` object to a BigQuery table. -.. literalinclude:: samples/to_gbq_simple.py +.. literalinclude:: samples/snippets/to_gbq_simple.py :language: python :dedent: 4 :start-after: [START bigquery_pandas_gbq_to_gbq_simple] diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index b8a3a985648f..ab50d5d71bb5 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -92,12 +92,12 @@ def cover(session): @nox.session(python=DEFAULT_PYTHON_VERSION) def docs(session): - """Build the docs.""" + """Build the docs for this library.""" - session.install("-r", os.path.join("docs", "requirements-docs.txt")) session.install("-e", ".") + session.install("sphinx==4.0.1", "alabaster", "recommonmark") - shutil.rmtree(os.path.join("docs", "source", "_build"), ignore_errors=True) + shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) session.run( "sphinx-build", "-W", # warnings as errors @@ -106,9 +106,9 @@ def docs(session): "-b", "html", "-d", - os.path.join("docs", "source", "_build", "doctrees", ""), - os.path.join("docs", "source", ""), - os.path.join("docs", "source", "_build", "html", ""), + os.path.join("docs", "_build", "doctrees", ""), + os.path.join("docs", ""), + os.path.join("docs", "_build", "html", ""), ) @@ -132,7 +132,7 @@ def system(session): session.run( "pytest", os.path.join(".", "tests", "system"), - os.path.join(".", "samples", "tests"), + os.path.join(".", "samples", "snippets"), "-v", *additional_args, ) diff --git a/packages/pandas-gbq/samples/samples b/packages/pandas-gbq/samples/samples deleted file mode 120000 index d9d97bb8bb7b..000000000000 --- a/packages/pandas-gbq/samples/samples +++ /dev/null @@ -1 +0,0 @@ -docs/source/samples \ No newline at end of file diff --git a/packages/pandas-gbq/samples/__init__.py b/packages/pandas-gbq/samples/snippets/__init__.py similarity index 100% rename from packages/pandas-gbq/samples/__init__.py rename to packages/pandas-gbq/samples/snippets/__init__.py diff --git a/packages/pandas-gbq/samples/tests/conftest.py b/packages/pandas-gbq/samples/snippets/conftest.py similarity index 100% rename from packages/pandas-gbq/samples/tests/conftest.py rename to packages/pandas-gbq/samples/snippets/conftest.py diff --git a/packages/pandas-gbq/samples/read_gbq_legacy.py b/packages/pandas-gbq/samples/snippets/read_gbq_legacy.py similarity index 100% rename from packages/pandas-gbq/samples/read_gbq_legacy.py rename to packages/pandas-gbq/samples/snippets/read_gbq_legacy.py diff --git a/packages/pandas-gbq/samples/read_gbq_simple.py b/packages/pandas-gbq/samples/snippets/read_gbq_simple.py similarity index 100% rename from packages/pandas-gbq/samples/read_gbq_simple.py rename to packages/pandas-gbq/samples/snippets/read_gbq_simple.py diff --git a/packages/pandas-gbq/samples/tests/test_read_gbq.py b/packages/pandas-gbq/samples/snippets/test_read_gbq.py similarity index 90% rename from packages/pandas-gbq/samples/tests/test_read_gbq.py rename to packages/pandas-gbq/samples/snippets/test_read_gbq.py index 1882ded0dd40..8f4992d71d58 100644 --- a/packages/pandas-gbq/samples/tests/test_read_gbq.py +++ b/packages/pandas-gbq/samples/snippets/test_read_gbq.py @@ -4,8 +4,8 @@ """System tests for read_gbq code samples.""" -from .. import read_gbq_legacy -from .. import read_gbq_simple +from . import read_gbq_legacy +from . import read_gbq_simple def test_read_gbq_legacy(project_id): diff --git a/packages/pandas-gbq/samples/tests/test_to_gbq.py b/packages/pandas-gbq/samples/snippets/test_to_gbq.py similarity index 93% rename from packages/pandas-gbq/samples/tests/test_to_gbq.py rename to packages/pandas-gbq/samples/snippets/test_to_gbq.py index c25b90591cc7..f42089742180 100644 --- a/packages/pandas-gbq/samples/tests/test_to_gbq.py +++ b/packages/pandas-gbq/samples/snippets/test_to_gbq.py @@ -4,7 +4,7 @@ """System tests for to_gbq code samples.""" -from .. import to_gbq_simple +from . import to_gbq_simple def test_to_gbq_simple(project_id, bigquery_client, random_dataset_id): diff --git a/packages/pandas-gbq/samples/to_gbq_simple.py b/packages/pandas-gbq/samples/snippets/to_gbq_simple.py similarity index 100% rename from packages/pandas-gbq/samples/to_gbq_simple.py rename to packages/pandas-gbq/samples/snippets/to_gbq_simple.py diff --git a/packages/pandas-gbq/samples/tests/__init__.py b/packages/pandas-gbq/samples/tests/__init__.py deleted file mode 100644 index c9ab850639a6..000000000000 --- a/packages/pandas-gbq/samples/tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# Copyright (c) 2017 pandas-gbq Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. From 4a5d8adf9a97cbfc52ff07bc3c819d78c5a7e465 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 29 Sep 2021 15:09:11 -0500 Subject: [PATCH 232/519] test: use templated noxfiles (#393) * test: use templated noxfiles Adjust test configuration to prepare for running on Kokoro. * enable system tests on Circle CI * use environment variables for project ID * add google-cloud-testutils to test deps * use pip script for circle * use pandas from constraints * fix constraints path * use nox * add ADC --- packages/pandas-gbq/.circleci/config.yml | 4 +- packages/pandas-gbq/ci/constraints-3.7.pip | 8 - packages/pandas-gbq/ci/run_pip.sh | 25 -- packages/pandas-gbq/ci/run_tests.sh | 2 +- packages/pandas-gbq/conftest.py | 80 ------ .../pandas-gbq/docs/howto/authentication.rst | 2 +- packages/pandas-gbq/noxfile.py | 187 +++++++++--- .../pandas-gbq/samples/snippets/conftest.py | 38 ++- .../pandas-gbq/samples/snippets/noxfile.py | 266 ++++++++++++++++++ .../{test_read_gbq.py => read_gbq_test.py} | 0 .../samples/snippets/requirements-test.txt | 2 + .../samples/snippets/requirements.txt | 5 + .../{test_to_gbq.py => to_gbq_test.py} | 6 +- .../pandas-gbq/testing/constraints-3.7.txt | 15 + .../constraints-3.8.txt} | 0 .../constraints-3.9.txt} | 0 packages/pandas-gbq/tests/__init__.py | 3 - packages/pandas-gbq/tests/system/conftest.py | 65 ++++- packages/pandas-gbq/tests/system/test_auth.py | 61 +--- .../system/test_read_gbq_with_bqstorage.py | 7 +- .../pandas-gbq/tests/system/test_to_gbq.py | 6 +- 21 files changed, 547 insertions(+), 235 deletions(-) delete mode 100644 packages/pandas-gbq/ci/constraints-3.7.pip delete mode 100755 packages/pandas-gbq/ci/run_pip.sh delete mode 100644 packages/pandas-gbq/conftest.py create mode 100644 packages/pandas-gbq/samples/snippets/noxfile.py rename packages/pandas-gbq/samples/snippets/{test_read_gbq.py => read_gbq_test.py} (100%) create mode 100644 packages/pandas-gbq/samples/snippets/requirements-test.txt create mode 100644 packages/pandas-gbq/samples/snippets/requirements.txt rename packages/pandas-gbq/samples/snippets/{test_to_gbq.py => to_gbq_test.py} (67%) create mode 100644 packages/pandas-gbq/testing/constraints-3.7.txt rename packages/pandas-gbq/{ci/constraints-3.8.pip => testing/constraints-3.8.txt} (100%) rename packages/pandas-gbq/{ci/constraints-3.9.pip => testing/constraints-3.9.txt} (100%) delete mode 100644 packages/pandas-gbq/tests/__init__.py diff --git a/packages/pandas-gbq/.circleci/config.yml b/packages/pandas-gbq/.circleci/config.yml index 1f9c1a42ef5d..626a4ed09e85 100644 --- a/packages/pandas-gbq/.circleci/config.yml +++ b/packages/pandas-gbq/.circleci/config.yml @@ -39,7 +39,7 @@ jobs: steps: - checkout - run: ci/config_auth.sh - - run: nox -s unit-3.8 system-3.8 cover + - run: export GOOGLE_APPLICATION_CREDENTIALS="$(pwd)/ci/service_account.json"; nox -s unit-3.8 system-3.8 cover # Conda "conda-3.7": @@ -72,4 +72,4 @@ workflows: - "pip-3.7" - "pip-3.8" - "conda-3.7" - - "conda-3.9-NIGHTLY" \ No newline at end of file + - "conda-3.9-NIGHTLY" diff --git a/packages/pandas-gbq/ci/constraints-3.7.pip b/packages/pandas-gbq/ci/constraints-3.7.pip deleted file mode 100644 index 362dfc850a5f..000000000000 --- a/packages/pandas-gbq/ci/constraints-3.7.pip +++ /dev/null @@ -1,8 +0,0 @@ -numpy==1.14.5 -pandas==0.23.2 -google-auth==1.4.1 -google-auth-oauthlib==0.0.1 -google-cloud-bigquery==1.11.1 -google-cloud-bigquery-storage==1.1.0 -pydata-google-auth==0.1.2 -tqdm==4.23.0 diff --git a/packages/pandas-gbq/ci/run_pip.sh b/packages/pandas-gbq/ci/run_pip.sh deleted file mode 100755 index 855b322e796d..000000000000 --- a/packages/pandas-gbq/ci/run_pip.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -# Copyright (c) 2017 pandas-gbq Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -set -e -x -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -# Install dependencies using Pip - -if [[ "$PANDAS" == "MASTER" ]]; then - PRE_WHEELS="https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com"; - pip install --pre --upgrade --timeout=60 -f $PRE_WHEELS pandas; -else - pip install pandas==$PANDAS -fi - -# Install test requirements -pip install coverage pytest pytest-cov flake8 codecov - -REQ="ci/requirements-${PYTHON}-${PANDAS}" -pip install -r "$REQ.pip" -pip install -e . - -$DIR/run_tests.sh diff --git a/packages/pandas-gbq/ci/run_tests.sh b/packages/pandas-gbq/ci/run_tests.sh index 3b0113bacdbd..efc12fe148a1 100755 --- a/packages/pandas-gbq/ci/run_tests.sh +++ b/packages/pandas-gbq/ci/run_tests.sh @@ -11,5 +11,5 @@ if [ -f "$DIR/service_account.json" ]; then fi # Install test requirements -pip install coverage pytest pytest-cov flake8 codecov +pip install coverage pytest pytest-cov flake8 codecov google-cloud-testutils pytest -v -m "not local_auth" --cov=pandas_gbq --cov-report xml:/tmp/pytest-cov.xml tests diff --git a/packages/pandas-gbq/conftest.py b/packages/pandas-gbq/conftest.py deleted file mode 100644 index ed6ebb0fc5ad..000000000000 --- a/packages/pandas-gbq/conftest.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright (c) 2017 pandas-gbq Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -"""Shared pytest fixtures for `tests/system` and `samples/tests` tests.""" - -import os -import os.path -import uuid - -import google.oauth2.service_account -import pytest - - -@pytest.fixture(scope="session") -def project_id(): - return os.environ.get("GBQ_PROJECT_ID") or os.environ.get( - "GOOGLE_CLOUD_PROJECT" - ) # noqa - - -@pytest.fixture(scope="session") -def private_key_path(): - path = os.path.join( - "ci", "service_account.json" - ) # Written by the 'ci/config_auth.sh' script. - if "GBQ_GOOGLE_APPLICATION_CREDENTIALS" in os.environ: - path = os.environ["GBQ_GOOGLE_APPLICATION_CREDENTIALS"] - elif "GOOGLE_APPLICATION_CREDENTIALS" in os.environ: - path = os.environ["GOOGLE_APPLICATION_CREDENTIALS"] - - if not os.path.isfile(path): - pytest.skip( - "Cannot run integration tests when there is " - "no file at the private key json file path" - ) - return None - - return path - - -@pytest.fixture(scope="session") -def private_key_contents(private_key_path): - if private_key_path is None: - return None - - with open(private_key_path) as f: - return f.read() - - -@pytest.fixture(scope="module") -def bigquery_client(project_id, private_key_path): - from google.cloud import bigquery - - return bigquery.Client.from_service_account_json( - private_key_path, project=project_id - ) - - -@pytest.fixture() -def random_dataset_id(bigquery_client): - import google.api_core.exceptions - from google.cloud import bigquery - - dataset_id = "".join(["pandas_gbq_", str(uuid.uuid4()).replace("-", "_")]) - dataset_ref = bigquery.DatasetReference( - bigquery_client.project, dataset_id - ) - yield dataset_id - try: - bigquery_client.delete_dataset(dataset_ref, delete_contents=True) - except google.api_core.exceptions.NotFound: - pass # Not all tests actually create a dataset - - -@pytest.fixture() -def credentials(private_key_path): - return google.oauth2.service_account.Credentials.from_service_account_file( - private_key_path - ) diff --git a/packages/pandas-gbq/docs/howto/authentication.rst b/packages/pandas-gbq/docs/howto/authentication.rst index d49632de5e28..877b11894ee2 100644 --- a/packages/pandas-gbq/docs/howto/authentication.rst +++ b/packages/pandas-gbq/docs/howto/authentication.rst @@ -61,7 +61,7 @@ authentication methods: If pandas-gbq does not find cached credentials, it prompts you to open a web browser, where you can grant pandas-gbq permissions to access your cloud resources. These credentials are only used locally. See the - :doc:`privacy policy ` for details. + :doc:`privacy policy <../privacy>` for details. Authenticating with a Service Account diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index ab50d5d71bb5..da42b7461800 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -1,14 +1,24 @@ -# Copyright (c) 2017 pandas-gbq Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -"""Nox test automation configuration. - -See: https://nox.readthedocs.io/en/latest/ -""" - +# -*- coding: utf-8 -*- +# +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generated by synthtool. DO NOT EDIT! + +from __future__ import absolute_import import os -import os.path +import pathlib import shutil import nox @@ -18,9 +28,21 @@ BLACK_PATHS = ["docs", "pandas_gbq", "tests", "noxfile.py", "setup.py"] DEFAULT_PYTHON_VERSION = "3.8" -SYSTEM_TEST_PYTHON_VERSIONS = ["3.8"] +SYSTEM_TEST_PYTHON_VERSIONS = ["3.8", "3.9"] UNIT_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9"] +CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() + +# 'docfx' is excluded since it only needs to run in 'docs-presubmit' +nox.options.sessions = [ + "unit", + "system", + "cover", + "lint", + "lint_setup_py", + "blacken", + "docs", +] # Error if a python version is missing nox.options.error_on_missing_interpreters = True @@ -29,6 +51,7 @@ @nox.session(python=DEFAULT_PYTHON_VERSION) def lint(session): """Run linters. + Returns a failure if the linters find linting errors or sufficiently serious code quality issues. """ @@ -55,32 +78,99 @@ def lint_setup_py(session): session.run("python", "setup.py", "check", "--restructuredtext", "--strict") -@nox.session(python=UNIT_TEST_PYTHON_VERSIONS) -def unit(session): - session.install("pytest", "pytest-cov") +def default(session): + # Install all test dependencies, then install this package in-place. + + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) session.install( - "-e", - ".", - # Use dependencies versions from constraints file. This enables testing - # across a more full range of versions of the dependencies. + "mock", + "asyncmock", + "pytest", + "pytest-cov", + "pytest-asyncio", "-c", - os.path.join(".", "ci", "constraints-{}.pip".format(session.python)), + constraints_path, ) + + session.install("-e", ".[tqdm]", "-c", constraints_path) + + # Run py.test against the unit tests. session.run( - "pytest", - os.path.join(".", "tests", "unit"), - "-v", + "py.test", + "--quiet", + f"--junitxml=unit_{session.python}_sponge_log.xml", "--cov=pandas_gbq", - "--cov=tests.unit", - "--cov-report", - "xml:/tmp/pytest-cov.xml", + "--cov=tests/unit", + "--cov-append", + "--cov-config=.coveragerc", + "--cov-report=", + "--cov-fail-under=0", + os.path.join("tests", "unit"), *session.posargs, ) +@nox.session(python=UNIT_TEST_PYTHON_VERSIONS) +def unit(session): + """Run the unit test suite.""" + default(session) + + +@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) +def system(session): + """Run the system test suite.""" + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) + system_test_path = os.path.join("tests", "system.py") + system_test_folder_path = os.path.join("tests", "system") + + # Check the value of `RUN_SYSTEM_TESTS` env var. It defaults to true. + if os.environ.get("RUN_SYSTEM_TESTS", "true") == "false": + session.skip("RUN_SYSTEM_TESTS is set to false, skipping") + # Install pyopenssl for mTLS testing. + if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") == "true": + session.install("pyopenssl") + + system_test_exists = os.path.exists(system_test_path) + system_test_folder_exists = os.path.exists(system_test_folder_path) + # Sanity check: only run tests if found. + if not system_test_exists and not system_test_folder_exists: + session.skip("System tests were not found") + + # Use pre-release gRPC for system tests. + session.install("--pre", "grpcio") + + # Install all test dependencies, then install this package into the + # virtualenv's dist-packages. + session.install("mock", "pytest", "google-cloud-testutils", "-c", constraints_path) + session.install("-e", ".[tqdm]", "-c", constraints_path) + + # Run py.test against the system tests. + if system_test_exists: + session.run( + "py.test", + "--quiet", + f"--junitxml=system_{session.python}_sponge_log.xml", + system_test_path, + *session.posargs, + ) + if system_test_folder_exists: + session.run( + "py.test", + "--quiet", + f"--junitxml=system_{session.python}_sponge_log.xml", + system_test_folder_path, + *session.posargs, + ) + + @nox.session(python=DEFAULT_PYTHON_VERSION) def cover(session): """Run the final coverage report. + This outputs the coverage report aggregating coverage from the unit test runs (not system test runs), and then erases coverage data. """ @@ -112,27 +202,36 @@ def docs(session): ) -@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) -def system(session): - session.install("pytest", "pytest-cov") +@nox.session(python=DEFAULT_PYTHON_VERSION) +def docfx(session): + """Build the docfx yaml files for this library.""" + + session.install("-e", ".") session.install( - "-e", - ".", - # Use dependencies versions from constraints file. This enables testing - # across a more full range of versions of the dependencies. - "-c", - os.path.join(".", "ci", "constraints-{}.pip".format(session.python)), + "sphinx==4.0.1", "alabaster", "recommonmark", "gcp-sphinx-docfx-yaml" ) - # Skip local auth tests on CI. - additional_args = list(session.posargs) - if "CIRCLECI" in os.environ: - additional_args = additional_args + ["-m", "not local_auth"] - + shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) session.run( - "pytest", - os.path.join(".", "tests", "system"), - os.path.join(".", "samples", "snippets"), - "-v", - *additional_args, + "sphinx-build", + "-T", # show full traceback on exception + "-N", # no colors + "-D", + ( + "extensions=sphinx.ext.autodoc," + "sphinx.ext.autosummary," + "docfx_yaml.extension," + "sphinx.ext.intersphinx," + "sphinx.ext.coverage," + "sphinx.ext.napoleon," + "sphinx.ext.todo," + "sphinx.ext.viewcode," + "recommonmark" + ), + "-b", + "html", + "-d", + os.path.join("docs", "_build", "doctrees", ""), + os.path.join("docs", ""), + os.path.join("docs", "_build", "html", ""), ) diff --git a/packages/pandas-gbq/samples/snippets/conftest.py b/packages/pandas-gbq/samples/snippets/conftest.py index 1733fb84842d..e5a3a7d9c174 100644 --- a/packages/pandas-gbq/samples/snippets/conftest.py +++ b/packages/pandas-gbq/samples/snippets/conftest.py @@ -2,12 +2,38 @@ # Distributed under BSD 3-Clause License. # See LICENSE.txt for details. -import os - +from google.cloud import bigquery import pytest +import test_utils.prefixer + + +prefixer = test_utils.prefixer.Prefixer("python-bigquery-pandas", "samples/snippets") + + +@pytest.fixture(scope="session", autouse=True) +def cleanup_datasets(bigquery_client: bigquery.Client): + for dataset in bigquery_client.list_datasets(): + if prefixer.should_cleanup(dataset.dataset_id): + bigquery_client.delete_dataset( + dataset, delete_contents=True, not_found_ok=True + ) + + +@pytest.fixture(scope="session") +def bigquery_client() -> bigquery.Client: + return bigquery.Client() + + +@pytest.fixture(scope="session") +def project_id(bigquery_client) -> str: + return bigquery_client.project -@pytest.fixture(autouse=True) -def default_credentials(private_key_path): - """Setup application default credentials for use in code samples.""" - os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = private_key_path +@pytest.fixture(scope="session") +def dataset_id(bigquery_client: bigquery.Client, project_id: str): + dataset_id = prefixer.create_prefix() + full_dataset_id = f"{project_id}.{dataset_id}" + dataset = bigquery.Dataset(full_dataset_id) + bigquery_client.create_dataset(dataset) + yield dataset_id + bigquery_client.delete_dataset(dataset, delete_contents=True, not_found_ok=True) diff --git a/packages/pandas-gbq/samples/snippets/noxfile.py b/packages/pandas-gbq/samples/snippets/noxfile.py new file mode 100644 index 000000000000..a57b633c0cb7 --- /dev/null +++ b/packages/pandas-gbq/samples/snippets/noxfile.py @@ -0,0 +1,266 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import os +from pathlib import Path +import sys +from typing import Callable, Dict, List, Optional + +import nox + + +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING +# DO NOT EDIT THIS FILE EVER! +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING + +BLACK_VERSION = "black==19.10b0" + +# Copy `noxfile_config.py` to your directory and modify it instead. + +# `TEST_CONFIG` dict is a configuration hook that allows users to +# modify the test configurations. The values here should be in sync +# with `noxfile_config.py`. Users will copy `noxfile_config.py` into +# their directory and modify it. + +TEST_CONFIG = { + # You can opt out from the test for specific Python versions. + "ignored_versions": [], + # Old samples are opted out of enforcing Python type hints + # All new samples should feature them + "enforce_type_hints": False, + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + # If you need to use a specific version of pip, + # change pip_version_override to the string representation + # of the version number, for example, "20.2.4" + "pip_version_override": None, + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + "envs": {}, +} + + +try: + # Ensure we can import noxfile_config in the project's directory. + sys.path.append(".") + from noxfile_config import TEST_CONFIG_OVERRIDE +except ImportError as e: + print("No user noxfile_config found: detail: {}".format(e)) + TEST_CONFIG_OVERRIDE = {} + +# Update the TEST_CONFIG with the user supplied values. +TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) + + +def get_pytest_env_vars() -> Dict[str, str]: + """Returns a dict for pytest invocation.""" + ret = {} + + # Override the GCLOUD_PROJECT and the alias. + env_key = TEST_CONFIG["gcloud_project_env"] + # This should error out if not set. + ret["GOOGLE_CLOUD_PROJECT"] = os.environ[env_key] + + # Apply user supplied envs. + ret.update(TEST_CONFIG["envs"]) + return ret + + +# DO NOT EDIT - automatically generated. +# All versions used to test samples. +ALL_VERSIONS = ["3.7", "3.8", "3.9"] + +# Any default versions that should be ignored. +IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] + +TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS]) + +INSTALL_LIBRARY_FROM_SOURCE = os.environ.get("INSTALL_LIBRARY_FROM_SOURCE", False) in ( + "True", + "true", +) +# +# Style Checks +# + + +def _determine_local_import_names(start_dir: str) -> List[str]: + """Determines all import names that should be considered "local". + + This is used when running the linter to insure that import order is + properly checked. + """ + file_ext_pairs = [os.path.splitext(path) for path in os.listdir(start_dir)] + return [ + basename + for basename, extension in file_ext_pairs + if extension == ".py" + or os.path.isdir(os.path.join(start_dir, basename)) + and basename not in ("__pycache__") + ] + + +# Linting with flake8. +# +# We ignore the following rules: +# E203: whitespace before ‘:’ +# E266: too many leading ‘#’ for block comment +# E501: line too long +# I202: Additional newline in a section of imports +# +# We also need to specify the rules which are ignored by default: +# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121'] +FLAKE8_COMMON_ARGS = [ + "--show-source", + "--builtin=gettext", + "--max-complexity=20", + "--import-order-style=google", + "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py", + "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202", + "--max-line-length=88", +] + + +@nox.session +def lint(session: nox.sessions.Session) -> None: + if not TEST_CONFIG["enforce_type_hints"]: + session.install("flake8", "flake8-import-order") + else: + session.install("flake8", "flake8-import-order", "flake8-annotations") + + local_names = _determine_local_import_names(".") + args = FLAKE8_COMMON_ARGS + [ + "--application-import-names", + ",".join(local_names), + ".", + ] + session.run("flake8", *args) + + +# +# Black +# + + +@nox.session +def blacken(session: nox.sessions.Session) -> None: + session.install(BLACK_VERSION) + python_files = [path for path in os.listdir(".") if path.endswith(".py")] + + session.run("black", *python_files) + + +# +# Sample Tests +# + + +PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] + + +def _session_tests( + session: nox.sessions.Session, post_install: Callable = None +) -> None: + if TEST_CONFIG["pip_version_override"]: + pip_version = TEST_CONFIG["pip_version_override"] + session.install(f"pip=={pip_version}") + """Runs py.test for a particular project.""" + if os.path.exists("requirements.txt"): + if os.path.exists("constraints.txt"): + session.install("-r", "requirements.txt", "-c", "constraints.txt") + else: + session.install("-r", "requirements.txt") + + if os.path.exists("requirements-test.txt"): + if os.path.exists("constraints-test.txt"): + session.install("-r", "requirements-test.txt", "-c", "constraints-test.txt") + else: + session.install("-r", "requirements-test.txt") + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars(), + ) + + +@nox.session(python=ALL_VERSIONS) +def py(session: nox.sessions.Session) -> None: + """Runs py.test for a sample using the specified version of Python.""" + if session.python in TESTED_VERSIONS: + _session_tests(session) + else: + session.skip( + "SKIPPED: {} tests are disabled for this sample.".format(session.python) + ) + + +# +# Readmegen +# + + +def _get_repo_root() -> Optional[str]: + """ Returns the root folder of the project. """ + # Get root of this repository. Assume we don't have directories nested deeper than 10 items. + p = Path(os.getcwd()) + for i in range(10): + if p is None: + break + if Path(p / ".git").exists(): + return str(p) + # .git is not available in repos cloned via Cloud Build + # setup.py is always in the library's root, so use that instead + # https://github.com/googleapis/synthtool/issues/792 + if Path(p / "setup.py").exists(): + return str(p) + p = p.parent + raise Exception("Unable to detect repository root.") + + +GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")]) + + +@nox.session +@nox.parametrize("path", GENERATED_READMES) +def readmegen(session: nox.sessions.Session, path: str) -> None: + """(Re-)generates the readme for a sample.""" + session.install("jinja2", "pyyaml") + dir_ = os.path.dirname(path) + + if os.path.exists(os.path.join(dir_, "requirements.txt")): + session.install("-r", os.path.join(dir_, "requirements.txt")) + + in_file = os.path.join(dir_, "README.rst.in") + session.run( + "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file + ) diff --git a/packages/pandas-gbq/samples/snippets/test_read_gbq.py b/packages/pandas-gbq/samples/snippets/read_gbq_test.py similarity index 100% rename from packages/pandas-gbq/samples/snippets/test_read_gbq.py rename to packages/pandas-gbq/samples/snippets/read_gbq_test.py diff --git a/packages/pandas-gbq/samples/snippets/requirements-test.txt b/packages/pandas-gbq/samples/snippets/requirements-test.txt new file mode 100644 index 000000000000..3bb560ddf8b7 --- /dev/null +++ b/packages/pandas-gbq/samples/snippets/requirements-test.txt @@ -0,0 +1,2 @@ +google-cloud-testutils==1.1.0 +pytest==6.2.5 diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt new file mode 100644 index 000000000000..cf8cecae3a64 --- /dev/null +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -0,0 +1,5 @@ +google-cloud-bigquery-storage==2.8.0 +google-cloud-bigquery==2.27.0 +pandas==1.3.1 +pandas-gbq==0.15.0 +pyarrow==5.0.0 diff --git a/packages/pandas-gbq/samples/snippets/test_to_gbq.py b/packages/pandas-gbq/samples/snippets/to_gbq_test.py similarity index 67% rename from packages/pandas-gbq/samples/snippets/test_to_gbq.py rename to packages/pandas-gbq/samples/snippets/to_gbq_test.py index f42089742180..6db15522a46b 100644 --- a/packages/pandas-gbq/samples/snippets/test_to_gbq.py +++ b/packages/pandas-gbq/samples/snippets/to_gbq_test.py @@ -4,11 +4,13 @@ """System tests for to_gbq code samples.""" +import random + from . import to_gbq_simple -def test_to_gbq_simple(project_id, bigquery_client, random_dataset_id): - table_id = "{}.to_gbq_simple".format(random_dataset_id) +def test_to_gbq_simple(project_id, bigquery_client, dataset_id): + table_id = f"{dataset_id}.to_gbq_simple_{random.randint(0, 999999)}" to_gbq_simple.main(project_id, table_id) table = bigquery_client.get_table(table_id) assert table.num_rows == 3 diff --git a/packages/pandas-gbq/testing/constraints-3.7.txt b/packages/pandas-gbq/testing/constraints-3.7.txt new file mode 100644 index 000000000000..251c81b4492a --- /dev/null +++ b/packages/pandas-gbq/testing/constraints-3.7.txt @@ -0,0 +1,15 @@ +# This constraints file is used to check that lower bounds +# are correct in setup.py +# List *all* library dependencies and extras in this file. +# Pin the version to the lower bound. +# +# e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", +# Then this file should have foo==1.14.0 +numpy==1.14.5 +pandas==0.23.2 +google-auth==1.4.1 +google-auth-oauthlib==0.0.1 +google-cloud-bigquery==1.11.1 +google-cloud-bigquery-storage==1.1.0 +pydata-google-auth==0.1.2 +tqdm==4.23.0 diff --git a/packages/pandas-gbq/ci/constraints-3.8.pip b/packages/pandas-gbq/testing/constraints-3.8.txt similarity index 100% rename from packages/pandas-gbq/ci/constraints-3.8.pip rename to packages/pandas-gbq/testing/constraints-3.8.txt diff --git a/packages/pandas-gbq/ci/constraints-3.9.pip b/packages/pandas-gbq/testing/constraints-3.9.txt similarity index 100% rename from packages/pandas-gbq/ci/constraints-3.9.pip rename to packages/pandas-gbq/testing/constraints-3.9.txt diff --git a/packages/pandas-gbq/tests/__init__.py b/packages/pandas-gbq/tests/__init__.py deleted file mode 100644 index c9ab850639a6..000000000000 --- a/packages/pandas-gbq/tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# Copyright (c) 2017 pandas-gbq Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. diff --git a/packages/pandas-gbq/tests/system/conftest.py b/packages/pandas-gbq/tests/system/conftest.py index 4745da0c7137..c29649029b80 100644 --- a/packages/pandas-gbq/tests/system/conftest.py +++ b/packages/pandas-gbq/tests/system/conftest.py @@ -2,22 +2,67 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -import google.oauth2.service_account +import os +import pathlib + +from google.cloud import bigquery import pytest +import test_utils.prefixer + + +prefixer = test_utils.prefixer.Prefixer("python-bigquery-pandas", "tests/system") + +REPO_DIR = pathlib.Path(__file__).parent.parent.parent + + +# TODO: remove when fully migrated off of Circle CI +@pytest.fixture(scope="session", autouse=True) +def default_credentials(): + """Setup application default credentials for use in code samples.""" + # Written by the 'ci/config_auth.sh' script. + path = REPO_DIR / "ci" / "service_account.json" + + if path.is_file() and "GOOGLE_APPLICATION_CREDENTIALS" not in os.environ: + os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = str(path) + + +@pytest.fixture(scope="session", autouse=True) +def cleanup_datasets(bigquery_client: bigquery.Client): + for dataset in bigquery_client.list_datasets(): + if prefixer.should_cleanup(dataset.dataset_id): + bigquery_client.delete_dataset( + dataset, delete_contents=True, not_found_ok=True + ) + + +@pytest.fixture(scope="session") +def bigquery_client() -> bigquery.Client: + project_id = os.getenv("GOOGLE_CLOUD_PROJECT", os.getenv("GBQ_PROJECT_ID")) + return bigquery.Client(project=project_id) + + +@pytest.fixture() +def credentials(bigquery_client): + return bigquery_client._credentials + + +@pytest.fixture(scope="session") +def project_id(bigquery_client) -> str: + return bigquery_client.project -@pytest.fixture(params=["env"]) -def project(request, project_id): - if request.param == "env": - return project_id - elif request.param == "none": - return None +@pytest.fixture(scope="session") +def project(project_id): + return project_id @pytest.fixture() -def credentials(private_key_path): - return google.oauth2.service_account.Credentials.from_service_account_file( - private_key_path +def random_dataset_id(bigquery_client: bigquery.Client, project_id: str): + dataset_id = prefixer.create_prefix() + full_dataset_id = f"{project_id}.{dataset_id}" + yield dataset_id + bigquery_client.delete_dataset( + full_dataset_id, delete_contents=True, not_found_ok=True ) diff --git a/packages/pandas-gbq/tests/system/test_auth.py b/packages/pandas-gbq/tests/system/test_auth.py index 34d5c8ffd253..d9f7d0965469 100644 --- a/packages/pandas-gbq/tests/system/test_auth.py +++ b/packages/pandas-gbq/tests/system/test_auth.py @@ -12,60 +12,21 @@ from pandas_gbq import auth -def mock_default_credentials(scopes=None, request=None): - return (None, None) - - -def _try_credentials(project_id, credentials): - from google.cloud import bigquery - import google.api_core.exceptions - import google.auth.exceptions - - if not credentials: - return None - if not project_id: - return credentials - - try: - client = bigquery.Client(project=project_id, credentials=credentials) - # Check if the application has rights to the BigQuery project - client.query("SELECT 1").result() - return credentials - except google.api_core.exceptions.GoogleAPIError: - return None - except google.auth.exceptions.RefreshError: - # Sometimes (such as on Travis) google-auth returns GCE credentials, - # but fetching the token for those credentials doesn't actually work. - # See: - # https://github.com/googleapis/google-auth-library-python/issues/287 - return None - - -def _check_if_can_get_correct_default_credentials(): - # Checks if "Application Default Credentials" can be fetched - # from the environment the tests are running in. - # See https://github.com/pandas-dev/pandas/issues/13577 +IS_RUNNING_ON_CI = "CIRCLE_BUILD_NUM" in os.environ or "KOKORO_BUILD_ID" in os.environ - import google.auth - from google.auth.exceptions import DefaultCredentialsError - import pandas_gbq.auth - import pandas_gbq.gbq - - try: - credentials, project = google.auth.default(scopes=pandas_gbq.auth.SCOPES) - except (DefaultCredentialsError, IOError): - return False - return _try_credentials(project, credentials) is not None +def mock_default_credentials(scopes=None, request=None): + return (None, None) -def test_should_be_able_to_get_valid_credentials(project_id, private_key_path): - os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = private_key_path +def test_should_be_able_to_get_valid_credentials(project_id): credentials, _ = auth.get_credentials(project_id=project_id) assert credentials.valid -@pytest.mark.local_auth +@pytest.mark.skipif( + IS_RUNNING_ON_CI, reason="end-user auth requires human intervention" +) def test_get_credentials_bad_file_returns_user_credentials(project_id, monkeypatch): import google.auth from google.auth.credentials import Credentials @@ -79,7 +40,9 @@ def test_get_credentials_bad_file_returns_user_credentials(project_id, monkeypat assert isinstance(credentials, Credentials) -@pytest.mark.local_auth +@pytest.mark.skipif( + IS_RUNNING_ON_CI, reason="end-user auth requires human intervention" +) def test_get_credentials_user_credentials_with_reauth(project_id, monkeypatch): import google.auth @@ -91,7 +54,9 @@ def test_get_credentials_user_credentials_with_reauth(project_id, monkeypatch): assert credentials.valid -@pytest.mark.local_auth +@pytest.mark.skipif( + IS_RUNNING_ON_CI, reason="end-user auth requires human intervention" +) def test_get_credentials_user_credentials(project_id, monkeypatch): import google.auth diff --git a/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py b/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py index 8440948a3e73..cddcedf0260f 100644 --- a/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py +++ b/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py @@ -14,10 +14,12 @@ @pytest.fixture -def method_under_test(credentials): +def method_under_test(project_id, credentials): import pandas_gbq - return functools.partial(pandas_gbq.read_gbq, credentials=credentials) + return functools.partial( + pandas_gbq.read_gbq, project_id=project_id, credentials=credentials + ) @pytest.mark.parametrize( @@ -38,7 +40,6 @@ def test_empty_results(method_under_test, query_string): assert len(df.index) == 0 -@pytest.mark.slow(reason="Large query for BQ Storage API tests.") def test_large_results(random_dataset, method_under_test): df = method_under_test( """ diff --git a/packages/pandas-gbq/tests/system/test_to_gbq.py b/packages/pandas-gbq/tests/system/test_to_gbq.py index f500942184ff..b0d2d031db8f 100644 --- a/packages/pandas-gbq/tests/system/test_to_gbq.py +++ b/packages/pandas-gbq/tests/system/test_to_gbq.py @@ -13,10 +13,12 @@ @pytest.fixture -def method_under_test(credentials): +def method_under_test(credentials, project_id): import pandas_gbq - return functools.partial(pandas_gbq.to_gbq, credentials=credentials) + return functools.partial( + pandas_gbq.to_gbq, project_id=project_id, credentials=credentials + ) def test_float_round_trip(method_under_test, random_dataset_id, bigquery_client): From ddc798614fa759d67a26639d30280e628e29e6db Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 5 Oct 2021 10:51:44 -0400 Subject: [PATCH 233/519] chore: add default_version and codeowner_team to .repo-metadata.json (#402) * chore: add default_version and codeowner_team to .repo-metadata.json * update codeowner_team --- packages/pandas-gbq/.repo-metadata.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/.repo-metadata.json b/packages/pandas-gbq/.repo-metadata.json index 72d7285adbff..fae1dc2089e5 100644 --- a/packages/pandas-gbq/.repo-metadata.json +++ b/packages/pandas-gbq/.repo-metadata.json @@ -9,5 +9,7 @@ "library_type": "INTEGRATION", "repo": "googleapis/python-bigquery-pandas", "distribution_name": "pandas-gbq", - "api_id": "bigquery.googleapis.com" - } + "api_id": "bigquery.googleapis.com", + "default_version": "", + "codeowner_team": "@googleapis/api-bigquery" +} From efea84cff76082227ca8521c80bc5cb0099f754f Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 5 Oct 2021 10:43:44 -0500 Subject: [PATCH 234/519] chore: enable owlbot and Kokoro (#380) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: use shared templates * add owlbot config * add owlbot lock file * fixup owlbot config * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * correct allowed headers * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * move constraints files * adjust coverage level * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * fix link to privacy document * link to pandas and pydata-google-auth docs * disable local auth tests on kokoro Keeps local_auth mark until we remove Circle config * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * simplify test auth * remove redundant jobs from circle * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * ignore Python 3.6 * remove coverage requirement from conda sessions * convert DatasetListItem to reference * use https for pandas intersphinx * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot Co-authored-by: Tres Seaver --- packages/pandas-gbq/.circleci/config.yml | 41 -- packages/pandas-gbq/.coveragerc | 38 ++ packages/pandas-gbq/.flake8 | 25 + packages/pandas-gbq/.github/.OwlBot.lock.yaml | 3 + packages/pandas-gbq/.github/.OwlBot.yaml | 19 + packages/pandas-gbq/.github/CONTRIBUTING.md | 28 + .../.github/ISSUE_TEMPLATE/bug_report.md | 43 ++ .../.github/ISSUE_TEMPLATE/feature_request.md | 18 + .../.github/ISSUE_TEMPLATE/support_request.md | 7 + .../.github/PULL_REQUEST_TEMPLATE.md | 10 +- .../.github/header-checker-lint.yml | 15 + packages/pandas-gbq/.github/snippet-bot.yml | 0 packages/pandas-gbq/.gitignore | 1 - packages/pandas-gbq/.kokoro/build.sh | 59 +++ .../pandas-gbq/.kokoro/continuous/common.cfg | 27 + .../.kokoro/continuous/continuous.cfg | 1 + .../pandas-gbq/.kokoro/docker/docs/Dockerfile | 67 +++ packages/pandas-gbq/.kokoro/docs/common.cfg | 65 +++ .../.kokoro/docs/docs-presubmit.cfg | 28 + packages/pandas-gbq/.kokoro/docs/docs.cfg | 1 + .../pandas-gbq/.kokoro/populate-secrets.sh | 43 ++ .../pandas-gbq/.kokoro/presubmit/common.cfg | 27 + .../.kokoro/presubmit/presubmit.cfg | 1 + packages/pandas-gbq/.kokoro/publish-docs.sh | 64 +++ packages/pandas-gbq/.kokoro/release.sh | 32 ++ .../pandas-gbq/.kokoro/release/common.cfg | 30 ++ .../pandas-gbq/.kokoro/release/release.cfg | 1 + .../.kokoro/samples/lint/common.cfg | 34 ++ .../.kokoro/samples/lint/continuous.cfg | 6 + .../.kokoro/samples/lint/periodic.cfg | 6 + .../.kokoro/samples/lint/presubmit.cfg | 6 + .../.kokoro/samples/python3.6/common.cfg | 40 ++ .../.kokoro/samples/python3.6/continuous.cfg | 7 + .../samples/python3.6/periodic-head.cfg | 11 + .../.kokoro/samples/python3.6/periodic.cfg | 6 + .../.kokoro/samples/python3.6/presubmit.cfg | 6 + .../.kokoro/samples/python3.7/common.cfg | 40 ++ .../.kokoro/samples/python3.7/continuous.cfg | 6 + .../samples/python3.7/periodic-head.cfg | 11 + .../.kokoro/samples/python3.7/periodic.cfg | 6 + .../.kokoro/samples/python3.7/presubmit.cfg | 6 + .../.kokoro/samples/python3.8/common.cfg | 40 ++ .../.kokoro/samples/python3.8/continuous.cfg | 6 + .../samples/python3.8/periodic-head.cfg | 11 + .../.kokoro/samples/python3.8/periodic.cfg | 6 + .../.kokoro/samples/python3.8/presubmit.cfg | 6 + .../.kokoro/samples/python3.9/common.cfg | 40 ++ .../.kokoro/samples/python3.9/continuous.cfg | 6 + .../samples/python3.9/periodic-head.cfg | 11 + .../.kokoro/samples/python3.9/periodic.cfg | 6 + .../.kokoro/samples/python3.9/presubmit.cfg | 6 + .../.kokoro/test-samples-against-head.sh | 28 + .../pandas-gbq/.kokoro/test-samples-impl.sh | 102 ++++ packages/pandas-gbq/.kokoro/test-samples.sh | 46 ++ packages/pandas-gbq/.kokoro/trampoline.sh | 28 + packages/pandas-gbq/.kokoro/trampoline_v2.sh | 487 ++++++++++++++++++ packages/pandas-gbq/.trampolinerc | 52 ++ packages/pandas-gbq/CODE_OF_CONDUCT.md | 9 +- packages/pandas-gbq/CONTRIBUTING.rst | 277 ++++++++++ packages/pandas-gbq/MANIFEST.in | 40 +- packages/pandas-gbq/ci/run_tests.sh | 2 +- packages/pandas-gbq/docs/_static/custom.css | 20 + .../pandas-gbq/docs/_templates/layout.html | 54 +- packages/pandas-gbq/noxfile.py | 2 +- packages/pandas-gbq/owlbot.py | 82 +++ packages/pandas-gbq/renovate.json | 11 +- .../pandas-gbq/samples/snippets/conftest.py | 2 +- .../pandas-gbq/samples/snippets/noxfile.py | 2 +- .../samples/snippets/noxfile_config.py | 7 + .../pandas-gbq/scripts/decrypt-secrets.sh | 46 ++ .../scripts/readme-gen/readme_gen.py | 66 +++ .../readme-gen/templates/README.tmpl.rst | 87 ++++ .../readme-gen/templates/auth.tmpl.rst | 9 + .../templates/auth_api_key.tmpl.rst | 14 + .../templates/install_deps.tmpl.rst | 29 ++ .../templates/install_portaudio.tmpl.rst | 35 ++ packages/pandas-gbq/testing/.gitignore | 3 + packages/pandas-gbq/tests/system/conftest.py | 2 +- 78 files changed, 2486 insertions(+), 79 deletions(-) create mode 100644 packages/pandas-gbq/.coveragerc create mode 100644 packages/pandas-gbq/.github/.OwlBot.lock.yaml create mode 100644 packages/pandas-gbq/.github/.OwlBot.yaml create mode 100644 packages/pandas-gbq/.github/CONTRIBUTING.md create mode 100644 packages/pandas-gbq/.github/ISSUE_TEMPLATE/bug_report.md create mode 100644 packages/pandas-gbq/.github/ISSUE_TEMPLATE/feature_request.md create mode 100644 packages/pandas-gbq/.github/ISSUE_TEMPLATE/support_request.md create mode 100644 packages/pandas-gbq/.github/header-checker-lint.yml create mode 100644 packages/pandas-gbq/.github/snippet-bot.yml create mode 100755 packages/pandas-gbq/.kokoro/build.sh create mode 100644 packages/pandas-gbq/.kokoro/continuous/common.cfg create mode 100644 packages/pandas-gbq/.kokoro/continuous/continuous.cfg create mode 100644 packages/pandas-gbq/.kokoro/docker/docs/Dockerfile create mode 100644 packages/pandas-gbq/.kokoro/docs/common.cfg create mode 100644 packages/pandas-gbq/.kokoro/docs/docs-presubmit.cfg create mode 100644 packages/pandas-gbq/.kokoro/docs/docs.cfg create mode 100755 packages/pandas-gbq/.kokoro/populate-secrets.sh create mode 100644 packages/pandas-gbq/.kokoro/presubmit/common.cfg create mode 100644 packages/pandas-gbq/.kokoro/presubmit/presubmit.cfg create mode 100755 packages/pandas-gbq/.kokoro/publish-docs.sh create mode 100755 packages/pandas-gbq/.kokoro/release.sh create mode 100644 packages/pandas-gbq/.kokoro/release/common.cfg create mode 100644 packages/pandas-gbq/.kokoro/release/release.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/lint/common.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/lint/continuous.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/lint/periodic.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/lint/presubmit.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.6/common.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.6/continuous.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.6/periodic-head.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.6/periodic.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.6/presubmit.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.7/common.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.7/continuous.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.7/periodic-head.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.7/periodic.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.7/presubmit.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.8/common.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.8/continuous.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.8/periodic-head.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.8/periodic.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.8/presubmit.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.9/common.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.9/continuous.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.9/periodic-head.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.9/periodic.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.9/presubmit.cfg create mode 100755 packages/pandas-gbq/.kokoro/test-samples-against-head.sh create mode 100755 packages/pandas-gbq/.kokoro/test-samples-impl.sh create mode 100755 packages/pandas-gbq/.kokoro/test-samples.sh create mode 100755 packages/pandas-gbq/.kokoro/trampoline.sh create mode 100755 packages/pandas-gbq/.kokoro/trampoline_v2.sh create mode 100644 packages/pandas-gbq/.trampolinerc create mode 100644 packages/pandas-gbq/CONTRIBUTING.rst create mode 100644 packages/pandas-gbq/docs/_static/custom.css create mode 100644 packages/pandas-gbq/owlbot.py create mode 100644 packages/pandas-gbq/samples/snippets/noxfile_config.py create mode 100755 packages/pandas-gbq/scripts/decrypt-secrets.sh create mode 100644 packages/pandas-gbq/scripts/readme-gen/readme_gen.py create mode 100644 packages/pandas-gbq/scripts/readme-gen/templates/README.tmpl.rst create mode 100644 packages/pandas-gbq/scripts/readme-gen/templates/auth.tmpl.rst create mode 100644 packages/pandas-gbq/scripts/readme-gen/templates/auth_api_key.tmpl.rst create mode 100644 packages/pandas-gbq/scripts/readme-gen/templates/install_deps.tmpl.rst create mode 100644 packages/pandas-gbq/scripts/readme-gen/templates/install_portaudio.tmpl.rst create mode 100644 packages/pandas-gbq/testing/.gitignore diff --git a/packages/pandas-gbq/.circleci/config.yml b/packages/pandas-gbq/.circleci/config.yml index 626a4ed09e85..ec4d7448a48f 100644 --- a/packages/pandas-gbq/.circleci/config.yml +++ b/packages/pandas-gbq/.circleci/config.yml @@ -4,43 +4,6 @@ version: 2 jobs: - "lint": - docker: - - image: thekevjames/nox - environment: - # Resolve "Python 3 was configured to use ASCII as encoding for the environment" - LC_ALL: C.UTF-8 - LANG: C.UTF-8 - steps: - - checkout - - run: nox -s lint - "docs-presubmit": - docker: - - image: thekevjames/nox - environment: - # Resolve "Python 3 was configured to use ASCII as encoding for the environment" - LC_ALL: C.UTF-8 - LANG: C.UTF-8 - steps: - - checkout - - run: nox -s docs - - # Pip - "pip-3.7": - docker: - - image: thekevjames/nox - steps: - - checkout - - run: ci/config_auth.sh - - run: nox -s unit-3.7 - "pip-3.8": - docker: - - image: thekevjames/nox - steps: - - checkout - - run: ci/config_auth.sh - - run: export GOOGLE_APPLICATION_CREDENTIALS="$(pwd)/ci/service_account.json"; nox -s unit-3.8 system-3.8 cover - # Conda "conda-3.7": docker: @@ -67,9 +30,5 @@ workflows: version: 2 build: jobs: - - lint - - docs-presubmit - - "pip-3.7" - - "pip-3.8" - "conda-3.7" - "conda-3.9-NIGHTLY" diff --git a/packages/pandas-gbq/.coveragerc b/packages/pandas-gbq/.coveragerc new file mode 100644 index 000000000000..0d8e6297dc9c --- /dev/null +++ b/packages/pandas-gbq/.coveragerc @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generated by synthtool. DO NOT EDIT! +[run] +branch = True +omit = + google/cloud/__init__.py + +[report] +fail_under = 100 +show_missing = True +exclude_lines = + # Re-enable the standard pragma + pragma: NO COVER + # Ignore debug-only repr + def __repr__ + # Ignore abstract methods + raise NotImplementedError +omit = + */gapic/*.py + */proto/*.py + */core/*.py + */site-packages/*.py + google/cloud/__init__.py diff --git a/packages/pandas-gbq/.flake8 b/packages/pandas-gbq/.flake8 index 0574e0a3ab66..29227d4cf419 100644 --- a/packages/pandas-gbq/.flake8 +++ b/packages/pandas-gbq/.flake8 @@ -1,7 +1,32 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generated by synthtool. DO NOT EDIT! [flake8] ignore = E203, E266, E501, W503 exclude = + # Exclude generated code. + **/proto/** + **/gapic/** + **/services/** + **/types/** + *_pb2.py + # Standard linting exemptions. + **/.nox/** __pycache__, .git, *.pyc, diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml new file mode 100644 index 000000000000..7b6cc31057ef --- /dev/null +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -0,0 +1,3 @@ +docker: + image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest + digest: sha256:a3a85c2e0b3293068e47b1635b178f7e3d3845f2cfb8722de6713d4bbafdcd1d diff --git a/packages/pandas-gbq/.github/.OwlBot.yaml b/packages/pandas-gbq/.github/.OwlBot.yaml new file mode 100644 index 000000000000..8642b5f3eaa4 --- /dev/null +++ b/packages/pandas-gbq/.github/.OwlBot.yaml @@ -0,0 +1,19 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker: + image: gcr.io/repo-automation-bots/owlbot-python:latest + +begin-after-commit-hash: 1afeb53252641dc35a421fa5acc59e2f3229ad6d + diff --git a/packages/pandas-gbq/.github/CONTRIBUTING.md b/packages/pandas-gbq/.github/CONTRIBUTING.md new file mode 100644 index 000000000000..939e5341e74d --- /dev/null +++ b/packages/pandas-gbq/.github/CONTRIBUTING.md @@ -0,0 +1,28 @@ +# How to Contribute + +We'd love to accept your patches and contributions to this project. There are +just a few small guidelines you need to follow. + +## Contributor License Agreement + +Contributions to this project must be accompanied by a Contributor License +Agreement. You (or your employer) retain the copyright to your contribution; +this simply gives us permission to use and redistribute your contributions as +part of the project. Head over to to see +your current agreements on file or to sign a new one. + +You generally only need to submit a CLA once, so if you've already submitted one +(even if it was for a different project), you probably don't need to do it +again. + +## Code reviews + +All submissions, including submissions by project members, require review. We +use GitHub pull requests for this purpose. Consult +[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more +information on using pull requests. + +## Community Guidelines + +This project follows [Google's Open Source Community +Guidelines](https://opensource.google.com/conduct/). diff --git a/packages/pandas-gbq/.github/ISSUE_TEMPLATE/bug_report.md b/packages/pandas-gbq/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000000..312388ed3446 --- /dev/null +++ b/packages/pandas-gbq/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,43 @@ +--- +name: Bug report +about: Create a report to help us improve + +--- + +Thanks for stopping by to let us know something could be better! + +**PLEASE READ**: If you have a support contract with Google, please create an issue in the [support console](https://cloud.google.com/support/) instead of filing on GitHub. This will ensure a timely response. + +Please run down the following list and make sure you've tried the usual "quick fixes": + + - Search the issues already opened: https://github.com/googleapis/python-bigquery-pandas/issues + - Search StackOverflow: https://stackoverflow.com/questions/tagged/google-cloud-platform+python + +If you are still having issues, please be sure to include as much information as possible: + +#### Environment details + + - OS type and version: + - Python version: `python --version` + - pip version: `pip --version` + - `pandas-gbq` version: `pip show pandas-gbq` + +#### Steps to reproduce + + 1. ? + 2. ? + +#### Code example + +```python +# example +``` + +#### Stack trace +``` +# example +``` + +Making sure to follow these steps will guarantee the quickest resolution possible. + +Thanks! diff --git a/packages/pandas-gbq/.github/ISSUE_TEMPLATE/feature_request.md b/packages/pandas-gbq/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000000..6365857f33c6 --- /dev/null +++ b/packages/pandas-gbq/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,18 @@ +--- +name: Feature request +about: Suggest an idea for this library + +--- + +Thanks for stopping by to let us know something could be better! + +**PLEASE READ**: If you have a support contract with Google, please create an issue in the [support console](https://cloud.google.com/support/) instead of filing on GitHub. This will ensure a timely response. + + **Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + **Describe the solution you'd like** +A clear and concise description of what you want to happen. + **Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + **Additional context** +Add any other context or screenshots about the feature request here. diff --git a/packages/pandas-gbq/.github/ISSUE_TEMPLATE/support_request.md b/packages/pandas-gbq/.github/ISSUE_TEMPLATE/support_request.md new file mode 100644 index 000000000000..995869032125 --- /dev/null +++ b/packages/pandas-gbq/.github/ISSUE_TEMPLATE/support_request.md @@ -0,0 +1,7 @@ +--- +name: Support request +about: If you have a support contract with Google, please create an issue in the Google Cloud Support console. + +--- + +**PLEASE READ**: If you have a support contract with Google, please create an issue in the [support console](https://cloud.google.com/support/) instead of filing on GitHub. This will ensure a timely response. diff --git a/packages/pandas-gbq/.github/PULL_REQUEST_TEMPLATE.md b/packages/pandas-gbq/.github/PULL_REQUEST_TEMPLATE.md index 872eb0ff8405..09b516833fa8 100644 --- a/packages/pandas-gbq/.github/PULL_REQUEST_TEMPLATE.md +++ b/packages/pandas-gbq/.github/PULL_REQUEST_TEMPLATE.md @@ -1,3 +1,7 @@ -- [ ] closes #xxxx -- [ ] tests added / passed -- [ ] passes `nox -s blacken lint` +Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: +- [ ] Make sure to open an issue as a [bug/issue](https://github.com/googleapis/python-bigquery-pandas/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea +- [ ] Ensure the tests and linter pass +- [ ] Code coverage does not decrease (if any source code was changed) +- [ ] Appropriate docs were updated (if necessary) + +Fixes # 🦕 diff --git a/packages/pandas-gbq/.github/header-checker-lint.yml b/packages/pandas-gbq/.github/header-checker-lint.yml new file mode 100644 index 000000000000..62cf51af9046 --- /dev/null +++ b/packages/pandas-gbq/.github/header-checker-lint.yml @@ -0,0 +1,15 @@ +{"allowedCopyrightHolders": ["pandas-gbq Authors"], + "allowedLicenses": ["Apache-2.0", "MIT", "BSD-3"], + "ignoreFiles": ["**/requirements.txt", "**/requirements-test.txt", "**/__init__.py", "samples/**/constraints.txt", "samples/**/constraints-test.txt"], + "sourceFileExtensions": [ + "ts", + "js", + "java", + "sh", + "Dockerfile", + "yaml", + "py", + "html", + "txt" + ] +} \ No newline at end of file diff --git a/packages/pandas-gbq/.github/snippet-bot.yml b/packages/pandas-gbq/.github/snippet-bot.yml new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/packages/pandas-gbq/.gitignore b/packages/pandas-gbq/.gitignore index 53f230ec0d56..b4243ced74e4 100644 --- a/packages/pandas-gbq/.gitignore +++ b/packages/pandas-gbq/.gitignore @@ -45,7 +45,6 @@ pip-log.txt # Built documentation docs/_build -docs/source/_build bigquery/docs/generated docs.metadata diff --git a/packages/pandas-gbq/.kokoro/build.sh b/packages/pandas-gbq/.kokoro/build.sh new file mode 100755 index 000000000000..d02a78db4dab --- /dev/null +++ b/packages/pandas-gbq/.kokoro/build.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +if [[ -z "${PROJECT_ROOT:-}" ]]; then + PROJECT_ROOT="github/python-bigquery-pandas" +fi + +cd "${PROJECT_ROOT}" + +# Disable buffering, so that the logs stream through. +export PYTHONUNBUFFERED=1 + +# Debug: show build environment +env | grep KOKORO + +# Setup service account credentials. +export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json + +# Setup project id. +export PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json") + +# Remove old nox +python3 -m pip uninstall --yes --quiet nox-automation + +# Install nox +python3 -m pip install --upgrade --quiet nox +python3 -m nox --version + +# If this is a continuous build, send the test log to the FlakyBot. +# See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot. +if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"continuous"* ]]; then + cleanup() { + chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot + $KOKORO_GFILE_DIR/linux_amd64/flakybot + } + trap cleanup EXIT HUP +fi + +# If NOX_SESSION is set, it only runs the specified session, +# otherwise run all the sessions. +if [[ -n "${NOX_SESSION:-}" ]]; then + python3 -m nox -s ${NOX_SESSION:-} +else + python3 -m nox +fi diff --git a/packages/pandas-gbq/.kokoro/continuous/common.cfg b/packages/pandas-gbq/.kokoro/continuous/common.cfg new file mode 100644 index 000000000000..4a34509712a2 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/continuous/common.cfg @@ -0,0 +1,27 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Download resources for system tests (service account key, etc.) +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-python" + +# Use the trampoline script to run in docker. +build_file: "python-bigquery-pandas/.kokoro/trampoline.sh" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-multi" +} +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-pandas/.kokoro/build.sh" +} diff --git a/packages/pandas-gbq/.kokoro/continuous/continuous.cfg b/packages/pandas-gbq/.kokoro/continuous/continuous.cfg new file mode 100644 index 000000000000..8f43917d92fe --- /dev/null +++ b/packages/pandas-gbq/.kokoro/continuous/continuous.cfg @@ -0,0 +1 @@ +# Format: //devtools/kokoro/config/proto/build.proto \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile b/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile new file mode 100644 index 000000000000..4e1b1fb8b5a5 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile @@ -0,0 +1,67 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ubuntu:20.04 + +ENV DEBIAN_FRONTEND noninteractive + +# Ensure local Python is preferred over distribution Python. +ENV PATH /usr/local/bin:$PATH + +# Install dependencies. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + apt-transport-https \ + build-essential \ + ca-certificates \ + curl \ + dirmngr \ + git \ + gpg-agent \ + graphviz \ + libbz2-dev \ + libdb5.3-dev \ + libexpat1-dev \ + libffi-dev \ + liblzma-dev \ + libreadline-dev \ + libsnappy-dev \ + libssl-dev \ + libsqlite3-dev \ + portaudio19-dev \ + python3-distutils \ + redis-server \ + software-properties-common \ + ssh \ + sudo \ + tcl \ + tcl-dev \ + tk \ + tk-dev \ + uuid-dev \ + wget \ + zlib1g-dev \ + && add-apt-repository universe \ + && apt-get update \ + && apt-get -y install jq \ + && apt-get clean autoclean \ + && apt-get autoremove -y \ + && rm -rf /var/lib/apt/lists/* \ + && rm -f /var/cache/apt/archives/*.deb + +RUN wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \ + && python3.8 /tmp/get-pip.py \ + && rm /tmp/get-pip.py + +CMD ["python3.8"] diff --git a/packages/pandas-gbq/.kokoro/docs/common.cfg b/packages/pandas-gbq/.kokoro/docs/common.cfg new file mode 100644 index 000000000000..9102163e5ab6 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/docs/common.cfg @@ -0,0 +1,65 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-bigquery-pandas/.kokoro/trampoline_v2.sh" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-lib-docs" +} +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-pandas/.kokoro/publish-docs.sh" +} + +env_vars: { + key: "STAGING_BUCKET" + value: "docs-staging" +} + +env_vars: { + key: "V2_STAGING_BUCKET" + value: "docs-staging-v2" +} + +# It will upload the docker image after successful builds. +env_vars: { + key: "TRAMPOLINE_IMAGE_UPLOAD" + value: "true" +} + +# It will always build the docker image. +env_vars: { + key: "TRAMPOLINE_DOCKERFILE" + value: ".kokoro/docker/docs/Dockerfile" +} + +# Fetch the token needed for reporting release status to GitHub +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "yoshi-automation-github-key" + } + } +} + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "docuploader_service_account" + } + } +} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/docs/docs-presubmit.cfg b/packages/pandas-gbq/.kokoro/docs/docs-presubmit.cfg new file mode 100644 index 000000000000..6c2a0fcf3195 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/docs/docs-presubmit.cfg @@ -0,0 +1,28 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "STAGING_BUCKET" + value: "gcloud-python-test" +} + +env_vars: { + key: "V2_STAGING_BUCKET" + value: "gcloud-python-test" +} + +# We only upload the image in the main `docs` build. +env_vars: { + key: "TRAMPOLINE_IMAGE_UPLOAD" + value: "false" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-pandas/.kokoro/build.sh" +} + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "docs docfx" +} diff --git a/packages/pandas-gbq/.kokoro/docs/docs.cfg b/packages/pandas-gbq/.kokoro/docs/docs.cfg new file mode 100644 index 000000000000..8f43917d92fe --- /dev/null +++ b/packages/pandas-gbq/.kokoro/docs/docs.cfg @@ -0,0 +1 @@ +# Format: //devtools/kokoro/config/proto/build.proto \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/populate-secrets.sh b/packages/pandas-gbq/.kokoro/populate-secrets.sh new file mode 100755 index 000000000000..f52514257ef0 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/populate-secrets.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Copyright 2020 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +function now { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n' ;} +function msg { println "$*" >&2 ;} +function println { printf '%s\n' "$(now) $*" ;} + + +# Populates requested secrets set in SECRET_MANAGER_KEYS from service account: +# kokoro-trampoline@cloud-devrel-kokoro-resources.iam.gserviceaccount.com +SECRET_LOCATION="${KOKORO_GFILE_DIR}/secret_manager" +msg "Creating folder on disk for secrets: ${SECRET_LOCATION}" +mkdir -p ${SECRET_LOCATION} +for key in $(echo ${SECRET_MANAGER_KEYS} | sed "s/,/ /g") +do + msg "Retrieving secret ${key}" + docker run --entrypoint=gcloud \ + --volume=${KOKORO_GFILE_DIR}:${KOKORO_GFILE_DIR} \ + gcr.io/google.com/cloudsdktool/cloud-sdk \ + secrets versions access latest \ + --project cloud-devrel-kokoro-resources \ + --secret ${key} > \ + "${SECRET_LOCATION}/${key}" + if [[ $? == 0 ]]; then + msg "Secret written to ${SECRET_LOCATION}/${key}" + else + msg "Error retrieving secret ${key}" + fi +done diff --git a/packages/pandas-gbq/.kokoro/presubmit/common.cfg b/packages/pandas-gbq/.kokoro/presubmit/common.cfg new file mode 100644 index 000000000000..4a34509712a2 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/presubmit/common.cfg @@ -0,0 +1,27 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Download resources for system tests (service account key, etc.) +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-python" + +# Use the trampoline script to run in docker. +build_file: "python-bigquery-pandas/.kokoro/trampoline.sh" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-multi" +} +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-pandas/.kokoro/build.sh" +} diff --git a/packages/pandas-gbq/.kokoro/presubmit/presubmit.cfg b/packages/pandas-gbq/.kokoro/presubmit/presubmit.cfg new file mode 100644 index 000000000000..8f43917d92fe --- /dev/null +++ b/packages/pandas-gbq/.kokoro/presubmit/presubmit.cfg @@ -0,0 +1 @@ +# Format: //devtools/kokoro/config/proto/build.proto \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/publish-docs.sh b/packages/pandas-gbq/.kokoro/publish-docs.sh new file mode 100755 index 000000000000..8acb14e802b0 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/publish-docs.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +# Disable buffering, so that the logs stream through. +export PYTHONUNBUFFERED=1 + +export PATH="${HOME}/.local/bin:${PATH}" + +# Install nox +python3 -m pip install --user --upgrade --quiet nox +python3 -m nox --version + +# build docs +nox -s docs + +python3 -m pip install --user gcp-docuploader + +# create metadata +python3 -m docuploader create-metadata \ + --name=$(jq --raw-output '.name // empty' .repo-metadata.json) \ + --version=$(python3 setup.py --version) \ + --language=$(jq --raw-output '.language // empty' .repo-metadata.json) \ + --distribution-name=$(python3 setup.py --name) \ + --product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \ + --github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \ + --issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json) + +cat docs.metadata + +# upload docs +python3 -m docuploader upload docs/_build/html --metadata-file docs.metadata --staging-bucket "${STAGING_BUCKET}" + + +# docfx yaml files +nox -s docfx + +# create metadata. +python3 -m docuploader create-metadata \ + --name=$(jq --raw-output '.name // empty' .repo-metadata.json) \ + --version=$(python3 setup.py --version) \ + --language=$(jq --raw-output '.language // empty' .repo-metadata.json) \ + --distribution-name=$(python3 setup.py --name) \ + --product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \ + --github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \ + --issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json) + +cat docs.metadata + +# upload docs +python3 -m docuploader upload docs/_build/html/docfx_yaml --metadata-file docs.metadata --destination-prefix docfx --staging-bucket "${V2_STAGING_BUCKET}" diff --git a/packages/pandas-gbq/.kokoro/release.sh b/packages/pandas-gbq/.kokoro/release.sh new file mode 100755 index 000000000000..10fe5b6f44f0 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/release.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +# Start the releasetool reporter +python3 -m pip install gcp-releasetool +python3 -m releasetool publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script + +# Ensure that we have the latest versions of Twine, Wheel, and Setuptools. +python3 -m pip install --upgrade twine wheel setuptools + +# Disable buffering, so that the logs stream through. +export PYTHONUNBUFFERED=1 + +# Move into the package, build the distribution and upload. +TWINE_PASSWORD=$(cat "${KOKORO_GFILE_DIR}/secret_manager/google-cloud-pypi-token") +cd github/python-bigquery-pandas +python3 setup.py sdist bdist_wheel +twine upload --username __token__ --password "${TWINE_PASSWORD}" dist/* diff --git a/packages/pandas-gbq/.kokoro/release/common.cfg b/packages/pandas-gbq/.kokoro/release/common.cfg new file mode 100644 index 000000000000..a67c994006e8 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/release/common.cfg @@ -0,0 +1,30 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-bigquery-pandas/.kokoro/trampoline.sh" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-multi" +} +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-pandas/.kokoro/release.sh" +} + +# Tokens needed to report release status back to GitHub +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem,google-cloud-pypi-token" +} diff --git a/packages/pandas-gbq/.kokoro/release/release.cfg b/packages/pandas-gbq/.kokoro/release/release.cfg new file mode 100644 index 000000000000..8f43917d92fe --- /dev/null +++ b/packages/pandas-gbq/.kokoro/release/release.cfg @@ -0,0 +1 @@ +# Format: //devtools/kokoro/config/proto/build.proto \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/lint/common.cfg b/packages/pandas-gbq/.kokoro/samples/lint/common.cfg new file mode 100644 index 000000000000..9e6f650c1dc0 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/lint/common.cfg @@ -0,0 +1,34 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "lint" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-pandas/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-bigquery-pandas/.kokoro/trampoline.sh" \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/lint/continuous.cfg b/packages/pandas-gbq/.kokoro/samples/lint/continuous.cfg new file mode 100644 index 000000000000..a1c8d9759c88 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/lint/continuous.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/lint/periodic.cfg b/packages/pandas-gbq/.kokoro/samples/lint/periodic.cfg new file mode 100644 index 000000000000..50fec9649732 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/lint/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/lint/presubmit.cfg b/packages/pandas-gbq/.kokoro/samples/lint/presubmit.cfg new file mode 100644 index 000000000000..a1c8d9759c88 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/lint/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.6/common.cfg b/packages/pandas-gbq/.kokoro/samples/python3.6/common.cfg new file mode 100644 index 000000000000..ced3d18c11c8 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.6/common.cfg @@ -0,0 +1,40 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.6" +} + +# Declare build specific Cloud project. +env_vars: { + key: "BUILD_SPECIFIC_GCLOUD_PROJECT" + value: "python-docs-samples-tests-py36" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-pandas/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-bigquery-pandas/.kokoro/trampoline.sh" \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.6/continuous.cfg b/packages/pandas-gbq/.kokoro/samples/python3.6/continuous.cfg new file mode 100644 index 000000000000..7218af1499e5 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.6/continuous.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + diff --git a/packages/pandas-gbq/.kokoro/samples/python3.6/periodic-head.cfg b/packages/pandas-gbq/.kokoro/samples/python3.6/periodic-head.cfg new file mode 100644 index 000000000000..98efde4dc99d --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.6/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-pandas/.kokoro/test-samples-against-head.sh" +} diff --git a/packages/pandas-gbq/.kokoro/samples/python3.6/periodic.cfg b/packages/pandas-gbq/.kokoro/samples/python3.6/periodic.cfg new file mode 100644 index 000000000000..50fec9649732 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.6/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.6/presubmit.cfg b/packages/pandas-gbq/.kokoro/samples/python3.6/presubmit.cfg new file mode 100644 index 000000000000..a1c8d9759c88 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.6/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.7/common.cfg b/packages/pandas-gbq/.kokoro/samples/python3.7/common.cfg new file mode 100644 index 000000000000..a61c609671d7 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.7/common.cfg @@ -0,0 +1,40 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.7" +} + +# Declare build specific Cloud project. +env_vars: { + key: "BUILD_SPECIFIC_GCLOUD_PROJECT" + value: "python-docs-samples-tests-py37" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-pandas/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-bigquery-pandas/.kokoro/trampoline.sh" \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.7/continuous.cfg b/packages/pandas-gbq/.kokoro/samples/python3.7/continuous.cfg new file mode 100644 index 000000000000..a1c8d9759c88 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.7/continuous.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.7/periodic-head.cfg b/packages/pandas-gbq/.kokoro/samples/python3.7/periodic-head.cfg new file mode 100644 index 000000000000..98efde4dc99d --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.7/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-pandas/.kokoro/test-samples-against-head.sh" +} diff --git a/packages/pandas-gbq/.kokoro/samples/python3.7/periodic.cfg b/packages/pandas-gbq/.kokoro/samples/python3.7/periodic.cfg new file mode 100644 index 000000000000..50fec9649732 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.7/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.7/presubmit.cfg b/packages/pandas-gbq/.kokoro/samples/python3.7/presubmit.cfg new file mode 100644 index 000000000000..a1c8d9759c88 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.7/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.8/common.cfg b/packages/pandas-gbq/.kokoro/samples/python3.8/common.cfg new file mode 100644 index 000000000000..75983c570b49 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.8/common.cfg @@ -0,0 +1,40 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.8" +} + +# Declare build specific Cloud project. +env_vars: { + key: "BUILD_SPECIFIC_GCLOUD_PROJECT" + value: "python-docs-samples-tests-py38" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-pandas/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-bigquery-pandas/.kokoro/trampoline.sh" \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.8/continuous.cfg b/packages/pandas-gbq/.kokoro/samples/python3.8/continuous.cfg new file mode 100644 index 000000000000..a1c8d9759c88 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.8/continuous.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.8/periodic-head.cfg b/packages/pandas-gbq/.kokoro/samples/python3.8/periodic-head.cfg new file mode 100644 index 000000000000..98efde4dc99d --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.8/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-pandas/.kokoro/test-samples-against-head.sh" +} diff --git a/packages/pandas-gbq/.kokoro/samples/python3.8/periodic.cfg b/packages/pandas-gbq/.kokoro/samples/python3.8/periodic.cfg new file mode 100644 index 000000000000..50fec9649732 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.8/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.8/presubmit.cfg b/packages/pandas-gbq/.kokoro/samples/python3.8/presubmit.cfg new file mode 100644 index 000000000000..a1c8d9759c88 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.8/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.9/common.cfg b/packages/pandas-gbq/.kokoro/samples/python3.9/common.cfg new file mode 100644 index 000000000000..0e88731298a0 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.9/common.cfg @@ -0,0 +1,40 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.9" +} + +# Declare build specific Cloud project. +env_vars: { + key: "BUILD_SPECIFIC_GCLOUD_PROJECT" + value: "python-docs-samples-tests-py39" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-pandas/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-bigquery-pandas/.kokoro/trampoline.sh" \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.9/continuous.cfg b/packages/pandas-gbq/.kokoro/samples/python3.9/continuous.cfg new file mode 100644 index 000000000000..a1c8d9759c88 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.9/continuous.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.9/periodic-head.cfg b/packages/pandas-gbq/.kokoro/samples/python3.9/periodic-head.cfg new file mode 100644 index 000000000000..98efde4dc99d --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.9/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-pandas/.kokoro/test-samples-against-head.sh" +} diff --git a/packages/pandas-gbq/.kokoro/samples/python3.9/periodic.cfg b/packages/pandas-gbq/.kokoro/samples/python3.9/periodic.cfg new file mode 100644 index 000000000000..50fec9649732 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.9/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.9/presubmit.cfg b/packages/pandas-gbq/.kokoro/samples/python3.9/presubmit.cfg new file mode 100644 index 000000000000..a1c8d9759c88 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.9/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/test-samples-against-head.sh b/packages/pandas-gbq/.kokoro/test-samples-against-head.sh new file mode 100755 index 000000000000..b382b13f8a88 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/test-samples-against-head.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A customized test runner for samples. +# +# For periodic builds, you can specify this file for testing against head. + +# `-e` enables the script to automatically fail when a command fails +# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero +set -eo pipefail +# Enables `**` to include files nested inside sub-folders +shopt -s globstar + +cd github/python-bigquery-pandas + +exec .kokoro/test-samples-impl.sh diff --git a/packages/pandas-gbq/.kokoro/test-samples-impl.sh b/packages/pandas-gbq/.kokoro/test-samples-impl.sh new file mode 100755 index 000000000000..8a324c9c7bc6 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/test-samples-impl.sh @@ -0,0 +1,102 @@ +#!/bin/bash +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# `-e` enables the script to automatically fail when a command fails +# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero +set -eo pipefail +# Enables `**` to include files nested inside sub-folders +shopt -s globstar + +# Exit early if samples don't exist +if ! find samples -name 'requirements.txt' | grep -q .; then + echo "No tests run. './samples/**/requirements.txt' not found" + exit 0 +fi + +# Disable buffering, so that the logs stream through. +export PYTHONUNBUFFERED=1 + +# Debug: show build environment +env | grep KOKORO + +# Install nox +python3.6 -m pip install --upgrade --quiet nox + +# Use secrets acessor service account to get secrets +if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then + gcloud auth activate-service-account \ + --key-file="${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" \ + --project="cloud-devrel-kokoro-resources" +fi + +# This script will create 3 files: +# - testing/test-env.sh +# - testing/service-account.json +# - testing/client-secrets.json +./scripts/decrypt-secrets.sh + +source ./testing/test-env.sh +export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/testing/service-account.json + +# For cloud-run session, we activate the service account for gcloud sdk. +gcloud auth activate-service-account \ + --key-file "${GOOGLE_APPLICATION_CREDENTIALS}" + +export GOOGLE_CLIENT_SECRETS=$(pwd)/testing/client-secrets.json + +echo -e "\n******************** TESTING PROJECTS ********************" + +# Switch to 'fail at end' to allow all tests to complete before exiting. +set +e +# Use RTN to return a non-zero value if the test fails. +RTN=0 +ROOT=$(pwd) +# Find all requirements.txt in the samples directory (may break on whitespace). +for file in samples/**/requirements.txt; do + cd "$ROOT" + # Navigate to the project folder. + file=$(dirname "$file") + cd "$file" + + echo "------------------------------------------------------------" + echo "- testing $file" + echo "------------------------------------------------------------" + + # Use nox to execute the tests for the project. + python3.6 -m nox -s "$RUN_TESTS_SESSION" + EXIT=$? + + # If this is a periodic build, send the test log to the FlakyBot. + # See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot. + if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then + chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot + $KOKORO_GFILE_DIR/linux_amd64/flakybot + fi + + if [[ $EXIT -ne 0 ]]; then + RTN=1 + echo -e "\n Testing failed: Nox returned a non-zero exit code. \n" + else + echo -e "\n Testing completed.\n" + fi + +done +cd "$ROOT" + +# Workaround for Kokoro permissions issue: delete secrets +rm testing/{test-env.sh,client-secrets.json,service-account.json} + +exit "$RTN" diff --git a/packages/pandas-gbq/.kokoro/test-samples.sh b/packages/pandas-gbq/.kokoro/test-samples.sh new file mode 100755 index 000000000000..9342d9522fa8 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/test-samples.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The default test runner for samples. +# +# For periodic builds, we rewinds the repo to the latest release, and +# run test-samples-impl.sh. + +# `-e` enables the script to automatically fail when a command fails +# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero +set -eo pipefail +# Enables `**` to include files nested inside sub-folders +shopt -s globstar + +cd github/python-bigquery-pandas + +# Run periodic samples tests at latest release +if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then + # preserving the test runner implementation. + cp .kokoro/test-samples-impl.sh "${TMPDIR}/test-samples-impl.sh" + echo "--- IMPORTANT IMPORTANT IMPORTANT ---" + echo "Now we rewind the repo back to the latest release..." + LATEST_RELEASE=$(git describe --abbrev=0 --tags) + git checkout $LATEST_RELEASE + echo "The current head is: " + echo $(git rev-parse --verify HEAD) + echo "--- IMPORTANT IMPORTANT IMPORTANT ---" + # move back the test runner implementation if there's no file. + if [ ! -f .kokoro/test-samples-impl.sh ]; then + cp "${TMPDIR}/test-samples-impl.sh" .kokoro/test-samples-impl.sh + fi +fi + +exec .kokoro/test-samples-impl.sh diff --git a/packages/pandas-gbq/.kokoro/trampoline.sh b/packages/pandas-gbq/.kokoro/trampoline.sh new file mode 100755 index 000000000000..f39236e943a8 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/trampoline.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +# Always run the cleanup script, regardless of the success of bouncing into +# the container. +function cleanup() { + chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh + ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh + echo "cleanup"; +} +trap cleanup EXIT + +$(dirname $0)/populate-secrets.sh # Secret Manager secrets. +python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py" \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/trampoline_v2.sh b/packages/pandas-gbq/.kokoro/trampoline_v2.sh new file mode 100755 index 000000000000..4af6cdc26dbc --- /dev/null +++ b/packages/pandas-gbq/.kokoro/trampoline_v2.sh @@ -0,0 +1,487 @@ +#!/usr/bin/env bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# trampoline_v2.sh +# +# This script does 3 things. +# +# 1. Prepare the Docker image for the test +# 2. Run the Docker with appropriate flags to run the test +# 3. Upload the newly built Docker image +# +# in a way that is somewhat compatible with trampoline_v1. +# +# To run this script, first download few files from gcs to /dev/shm. +# (/dev/shm is passed into the container as KOKORO_GFILE_DIR). +# +# gsutil cp gs://cloud-devrel-kokoro-resources/python-docs-samples/secrets_viewer_service_account.json /dev/shm +# gsutil cp gs://cloud-devrel-kokoro-resources/python-docs-samples/automl_secrets.txt /dev/shm +# +# Then run the script. +# .kokoro/trampoline_v2.sh +# +# These environment variables are required: +# TRAMPOLINE_IMAGE: The docker image to use. +# TRAMPOLINE_DOCKERFILE: The location of the Dockerfile. +# +# You can optionally change these environment variables: +# TRAMPOLINE_IMAGE_UPLOAD: +# (true|false): Whether to upload the Docker image after the +# successful builds. +# TRAMPOLINE_BUILD_FILE: The script to run in the docker container. +# TRAMPOLINE_WORKSPACE: The workspace path in the docker container. +# Defaults to /workspace. +# Potentially there are some repo specific envvars in .trampolinerc in +# the project root. + + +set -euo pipefail + +TRAMPOLINE_VERSION="2.0.5" + +if command -v tput >/dev/null && [[ -n "${TERM:-}" ]]; then + readonly IO_COLOR_RED="$(tput setaf 1)" + readonly IO_COLOR_GREEN="$(tput setaf 2)" + readonly IO_COLOR_YELLOW="$(tput setaf 3)" + readonly IO_COLOR_RESET="$(tput sgr0)" +else + readonly IO_COLOR_RED="" + readonly IO_COLOR_GREEN="" + readonly IO_COLOR_YELLOW="" + readonly IO_COLOR_RESET="" +fi + +function function_exists { + [ $(LC_ALL=C type -t $1)"" == "function" ] +} + +# Logs a message using the given color. The first argument must be one +# of the IO_COLOR_* variables defined above, such as +# "${IO_COLOR_YELLOW}". The remaining arguments will be logged in the +# given color. The log message will also have an RFC-3339 timestamp +# prepended (in UTC). You can disable the color output by setting +# TERM=vt100. +function log_impl() { + local color="$1" + shift + local timestamp="$(date -u "+%Y-%m-%dT%H:%M:%SZ")" + echo "================================================================" + echo "${color}${timestamp}:" "$@" "${IO_COLOR_RESET}" + echo "================================================================" +} + +# Logs the given message with normal coloring and a timestamp. +function log() { + log_impl "${IO_COLOR_RESET}" "$@" +} + +# Logs the given message in green with a timestamp. +function log_green() { + log_impl "${IO_COLOR_GREEN}" "$@" +} + +# Logs the given message in yellow with a timestamp. +function log_yellow() { + log_impl "${IO_COLOR_YELLOW}" "$@" +} + +# Logs the given message in red with a timestamp. +function log_red() { + log_impl "${IO_COLOR_RED}" "$@" +} + +readonly tmpdir=$(mktemp -d -t ci-XXXXXXXX) +readonly tmphome="${tmpdir}/h" +mkdir -p "${tmphome}" + +function cleanup() { + rm -rf "${tmpdir}" +} +trap cleanup EXIT + +RUNNING_IN_CI="${RUNNING_IN_CI:-false}" + +# The workspace in the container, defaults to /workspace. +TRAMPOLINE_WORKSPACE="${TRAMPOLINE_WORKSPACE:-/workspace}" + +pass_down_envvars=( + # TRAMPOLINE_V2 variables. + # Tells scripts whether they are running as part of CI or not. + "RUNNING_IN_CI" + # Indicates which CI system we're in. + "TRAMPOLINE_CI" + # Indicates the version of the script. + "TRAMPOLINE_VERSION" +) + +log_yellow "Building with Trampoline ${TRAMPOLINE_VERSION}" + +# Detect which CI systems we're in. If we're in any of the CI systems +# we support, `RUNNING_IN_CI` will be true and `TRAMPOLINE_CI` will be +# the name of the CI system. Both envvars will be passing down to the +# container for telling which CI system we're in. +if [[ -n "${KOKORO_BUILD_ID:-}" ]]; then + # descriptive env var for indicating it's on CI. + RUNNING_IN_CI="true" + TRAMPOLINE_CI="kokoro" + if [[ "${TRAMPOLINE_USE_LEGACY_SERVICE_ACCOUNT:-}" == "true" ]]; then + if [[ ! -f "${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json" ]]; then + log_red "${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json does not exist. Did you forget to mount cloud-devrel-kokoro-resources/trampoline? Aborting." + exit 1 + fi + # This service account will be activated later. + TRAMPOLINE_SERVICE_ACCOUNT="${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json" + else + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + gcloud auth list + fi + log_yellow "Configuring Container Registry access" + gcloud auth configure-docker --quiet + fi + pass_down_envvars+=( + # KOKORO dynamic variables. + "KOKORO_BUILD_NUMBER" + "KOKORO_BUILD_ID" + "KOKORO_JOB_NAME" + "KOKORO_GIT_COMMIT" + "KOKORO_GITHUB_COMMIT" + "KOKORO_GITHUB_PULL_REQUEST_NUMBER" + "KOKORO_GITHUB_PULL_REQUEST_COMMIT" + # For FlakyBot + "KOKORO_GITHUB_COMMIT_URL" + "KOKORO_GITHUB_PULL_REQUEST_URL" + ) +elif [[ "${TRAVIS:-}" == "true" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="travis" + pass_down_envvars+=( + "TRAVIS_BRANCH" + "TRAVIS_BUILD_ID" + "TRAVIS_BUILD_NUMBER" + "TRAVIS_BUILD_WEB_URL" + "TRAVIS_COMMIT" + "TRAVIS_COMMIT_MESSAGE" + "TRAVIS_COMMIT_RANGE" + "TRAVIS_JOB_NAME" + "TRAVIS_JOB_NUMBER" + "TRAVIS_JOB_WEB_URL" + "TRAVIS_PULL_REQUEST" + "TRAVIS_PULL_REQUEST_BRANCH" + "TRAVIS_PULL_REQUEST_SHA" + "TRAVIS_PULL_REQUEST_SLUG" + "TRAVIS_REPO_SLUG" + "TRAVIS_SECURE_ENV_VARS" + "TRAVIS_TAG" + ) +elif [[ -n "${GITHUB_RUN_ID:-}" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="github-workflow" + pass_down_envvars+=( + "GITHUB_WORKFLOW" + "GITHUB_RUN_ID" + "GITHUB_RUN_NUMBER" + "GITHUB_ACTION" + "GITHUB_ACTIONS" + "GITHUB_ACTOR" + "GITHUB_REPOSITORY" + "GITHUB_EVENT_NAME" + "GITHUB_EVENT_PATH" + "GITHUB_SHA" + "GITHUB_REF" + "GITHUB_HEAD_REF" + "GITHUB_BASE_REF" + ) +elif [[ "${CIRCLECI:-}" == "true" ]]; then + RUNNING_IN_CI="true" + TRAMPOLINE_CI="circleci" + pass_down_envvars+=( + "CIRCLE_BRANCH" + "CIRCLE_BUILD_NUM" + "CIRCLE_BUILD_URL" + "CIRCLE_COMPARE_URL" + "CIRCLE_JOB" + "CIRCLE_NODE_INDEX" + "CIRCLE_NODE_TOTAL" + "CIRCLE_PREVIOUS_BUILD_NUM" + "CIRCLE_PROJECT_REPONAME" + "CIRCLE_PROJECT_USERNAME" + "CIRCLE_REPOSITORY_URL" + "CIRCLE_SHA1" + "CIRCLE_STAGE" + "CIRCLE_USERNAME" + "CIRCLE_WORKFLOW_ID" + "CIRCLE_WORKFLOW_JOB_ID" + "CIRCLE_WORKFLOW_UPSTREAM_JOB_IDS" + "CIRCLE_WORKFLOW_WORKSPACE_ID" + ) +fi + +# Configure the service account for pulling the docker image. +function repo_root() { + local dir="$1" + while [[ ! -d "${dir}/.git" ]]; do + dir="$(dirname "$dir")" + done + echo "${dir}" +} + +# Detect the project root. In CI builds, we assume the script is in +# the git tree and traverse from there, otherwise, traverse from `pwd` +# to find `.git` directory. +if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + PROGRAM_PATH="$(realpath "$0")" + PROGRAM_DIR="$(dirname "${PROGRAM_PATH}")" + PROJECT_ROOT="$(repo_root "${PROGRAM_DIR}")" +else + PROJECT_ROOT="$(repo_root $(pwd))" +fi + +log_yellow "Changing to the project root: ${PROJECT_ROOT}." +cd "${PROJECT_ROOT}" + +# To support relative path for `TRAMPOLINE_SERVICE_ACCOUNT`, we need +# to use this environment variable in `PROJECT_ROOT`. +if [[ -n "${TRAMPOLINE_SERVICE_ACCOUNT:-}" ]]; then + + mkdir -p "${tmpdir}/gcloud" + gcloud_config_dir="${tmpdir}/gcloud" + + log_yellow "Using isolated gcloud config: ${gcloud_config_dir}." + export CLOUDSDK_CONFIG="${gcloud_config_dir}" + + log_yellow "Using ${TRAMPOLINE_SERVICE_ACCOUNT} for authentication." + gcloud auth activate-service-account \ + --key-file "${TRAMPOLINE_SERVICE_ACCOUNT}" + log_yellow "Configuring Container Registry access" + gcloud auth configure-docker --quiet +fi + +required_envvars=( + # The basic trampoline configurations. + "TRAMPOLINE_IMAGE" + "TRAMPOLINE_BUILD_FILE" +) + +if [[ -f "${PROJECT_ROOT}/.trampolinerc" ]]; then + source "${PROJECT_ROOT}/.trampolinerc" +fi + +log_yellow "Checking environment variables." +for e in "${required_envvars[@]}" +do + if [[ -z "${!e:-}" ]]; then + log "Missing ${e} env var. Aborting." + exit 1 + fi +done + +# We want to support legacy style TRAMPOLINE_BUILD_FILE used with V1 +# script: e.g. "github/repo-name/.kokoro/run_tests.sh" +TRAMPOLINE_BUILD_FILE="${TRAMPOLINE_BUILD_FILE#github/*/}" +log_yellow "Using TRAMPOLINE_BUILD_FILE: ${TRAMPOLINE_BUILD_FILE}" + +# ignore error on docker operations and test execution +set +e + +log_yellow "Preparing Docker image." +# We only download the docker image in CI builds. +if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + # Download the docker image specified by `TRAMPOLINE_IMAGE` + + # We may want to add --max-concurrent-downloads flag. + + log_yellow "Start pulling the Docker image: ${TRAMPOLINE_IMAGE}." + if docker pull "${TRAMPOLINE_IMAGE}"; then + log_green "Finished pulling the Docker image: ${TRAMPOLINE_IMAGE}." + has_image="true" + else + log_red "Failed pulling the Docker image: ${TRAMPOLINE_IMAGE}." + has_image="false" + fi +else + # For local run, check if we have the image. + if docker images "${TRAMPOLINE_IMAGE}:latest" | grep "${TRAMPOLINE_IMAGE}"; then + has_image="true" + else + has_image="false" + fi +fi + + +# The default user for a Docker container has uid 0 (root). To avoid +# creating root-owned files in the build directory we tell docker to +# use the current user ID. +user_uid="$(id -u)" +user_gid="$(id -g)" +user_name="$(id -un)" + +# To allow docker in docker, we add the user to the docker group in +# the host os. +docker_gid=$(cut -d: -f3 < <(getent group docker)) + +update_cache="false" +if [[ "${TRAMPOLINE_DOCKERFILE:-none}" != "none" ]]; then + # Build the Docker image from the source. + context_dir=$(dirname "${TRAMPOLINE_DOCKERFILE}") + docker_build_flags=( + "-f" "${TRAMPOLINE_DOCKERFILE}" + "-t" "${TRAMPOLINE_IMAGE}" + "--build-arg" "UID=${user_uid}" + "--build-arg" "USERNAME=${user_name}" + ) + if [[ "${has_image}" == "true" ]]; then + docker_build_flags+=("--cache-from" "${TRAMPOLINE_IMAGE}") + fi + + log_yellow "Start building the docker image." + if [[ "${TRAMPOLINE_VERBOSE:-false}" == "true" ]]; then + echo "docker build" "${docker_build_flags[@]}" "${context_dir}" + fi + + # ON CI systems, we want to suppress docker build logs, only + # output the logs when it fails. + if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then + if docker build "${docker_build_flags[@]}" "${context_dir}" \ + > "${tmpdir}/docker_build.log" 2>&1; then + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + cat "${tmpdir}/docker_build.log" + fi + + log_green "Finished building the docker image." + update_cache="true" + else + log_red "Failed to build the Docker image, aborting." + log_yellow "Dumping the build logs:" + cat "${tmpdir}/docker_build.log" + exit 1 + fi + else + if docker build "${docker_build_flags[@]}" "${context_dir}"; then + log_green "Finished building the docker image." + update_cache="true" + else + log_red "Failed to build the Docker image, aborting." + exit 1 + fi + fi +else + if [[ "${has_image}" != "true" ]]; then + log_red "We do not have ${TRAMPOLINE_IMAGE} locally, aborting." + exit 1 + fi +fi + +# We use an array for the flags so they are easier to document. +docker_flags=( + # Remove the container after it exists. + "--rm" + + # Use the host network. + "--network=host" + + # Run in priviledged mode. We are not using docker for sandboxing or + # isolation, just for packaging our dev tools. + "--privileged" + + # Run the docker script with the user id. Because the docker image gets to + # write in ${PWD} you typically want this to be your user id. + # To allow docker in docker, we need to use docker gid on the host. + "--user" "${user_uid}:${docker_gid}" + + # Pass down the USER. + "--env" "USER=${user_name}" + + # Mount the project directory inside the Docker container. + "--volume" "${PROJECT_ROOT}:${TRAMPOLINE_WORKSPACE}" + "--workdir" "${TRAMPOLINE_WORKSPACE}" + "--env" "PROJECT_ROOT=${TRAMPOLINE_WORKSPACE}" + + # Mount the temporary home directory. + "--volume" "${tmphome}:/h" + "--env" "HOME=/h" + + # Allow docker in docker. + "--volume" "/var/run/docker.sock:/var/run/docker.sock" + + # Mount the /tmp so that docker in docker can mount the files + # there correctly. + "--volume" "/tmp:/tmp" + # Pass down the KOKORO_GFILE_DIR and KOKORO_KEYSTORE_DIR + # TODO(tmatsuo): This part is not portable. + "--env" "TRAMPOLINE_SECRET_DIR=/secrets" + "--volume" "${KOKORO_GFILE_DIR:-/dev/shm}:/secrets/gfile" + "--env" "KOKORO_GFILE_DIR=/secrets/gfile" + "--volume" "${KOKORO_KEYSTORE_DIR:-/dev/shm}:/secrets/keystore" + "--env" "KOKORO_KEYSTORE_DIR=/secrets/keystore" +) + +# Add an option for nicer output if the build gets a tty. +if [[ -t 0 ]]; then + docker_flags+=("-it") +fi + +# Passing down env vars +for e in "${pass_down_envvars[@]}" +do + if [[ -n "${!e:-}" ]]; then + docker_flags+=("--env" "${e}=${!e}") + fi +done + +# If arguments are given, all arguments will become the commands run +# in the container, otherwise run TRAMPOLINE_BUILD_FILE. +if [[ $# -ge 1 ]]; then + log_yellow "Running the given commands '" "${@:1}" "' in the container." + readonly commands=("${@:1}") + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}" + fi + docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}" +else + log_yellow "Running the tests in a Docker container." + docker_flags+=("--entrypoint=${TRAMPOLINE_BUILD_FILE}") + if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then + echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" + fi + docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" +fi + + +test_retval=$? + +if [[ ${test_retval} -eq 0 ]]; then + log_green "Build finished with ${test_retval}" +else + log_red "Build finished with ${test_retval}" +fi + +# Only upload it when the test passes. +if [[ "${update_cache}" == "true" ]] && \ + [[ $test_retval == 0 ]] && \ + [[ "${TRAMPOLINE_IMAGE_UPLOAD:-false}" == "true" ]]; then + log_yellow "Uploading the Docker image." + if docker push "${TRAMPOLINE_IMAGE}"; then + log_green "Finished uploading the Docker image." + else + log_red "Failed uploading the Docker image." + fi + # Call trampoline_after_upload_hook if it's defined. + if function_exists trampoline_after_upload_hook; then + trampoline_after_upload_hook + fi + +fi + +exit "${test_retval}" diff --git a/packages/pandas-gbq/.trampolinerc b/packages/pandas-gbq/.trampolinerc new file mode 100644 index 000000000000..383b6ec89fbc --- /dev/null +++ b/packages/pandas-gbq/.trampolinerc @@ -0,0 +1,52 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Template for .trampolinerc + +# Add required env vars here. +required_envvars+=( + "STAGING_BUCKET" + "V2_STAGING_BUCKET" +) + +# Add env vars which are passed down into the container here. +pass_down_envvars+=( + "STAGING_BUCKET" + "V2_STAGING_BUCKET" + "NOX_SESSION" +) + +# Prevent unintentional override on the default image. +if [[ "${TRAMPOLINE_IMAGE_UPLOAD:-false}" == "true" ]] && \ + [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then + echo "Please set TRAMPOLINE_IMAGE if you want to upload the Docker image." + exit 1 +fi + +# Define the default value if it makes sense. +if [[ -z "${TRAMPOLINE_IMAGE_UPLOAD:-}" ]]; then + TRAMPOLINE_IMAGE_UPLOAD="" +fi + +if [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then + TRAMPOLINE_IMAGE="" +fi + +if [[ -z "${TRAMPOLINE_DOCKERFILE:-}" ]]; then + TRAMPOLINE_DOCKERFILE="" +fi + +if [[ -z "${TRAMPOLINE_BUILD_FILE:-}" ]]; then + TRAMPOLINE_BUILD_FILE="" +fi diff --git a/packages/pandas-gbq/CODE_OF_CONDUCT.md b/packages/pandas-gbq/CODE_OF_CONDUCT.md index 0bd87614b1cd..039f43681204 100644 --- a/packages/pandas-gbq/CODE_OF_CONDUCT.md +++ b/packages/pandas-gbq/CODE_OF_CONDUCT.md @@ -1,3 +1,4 @@ + # Code of Conduct ## Our Pledge @@ -69,8 +70,9 @@ dispute. If you are unable to resolve the matter for any reason, or if the behavior is threatening or harassing, report it. We are dedicated to providing an environment where participants feel welcome and safe. -Reports should be directed to *[PROJECT STEWARD NAME(s) AND EMAIL(s)]*, the -Project Steward(s) for *[PROJECT NAME]*. It is the Project Steward’s duty to + +Reports should be directed to *googleapis-stewards@google.com*, the +Project Steward(s) for *Google Cloud Client Libraries*. It is the Project Steward’s duty to receive and address reported violations of the code of conduct. They will then work with a committee consisting of representatives from the Open Source Programs Office and the Google Open Source Strategy team. If for any reason you @@ -90,5 +92,4 @@ harassment or threats to anyone's safety, we may take action without notice. This Code of Conduct is adapted from the Contributor Covenant, version 1.4, available at -https://www.contributor-covenant.org/version/1/4/code-of-conduct.html - +https://www.contributor-covenant.org/version/1/4/code-of-conduct.html \ No newline at end of file diff --git a/packages/pandas-gbq/CONTRIBUTING.rst b/packages/pandas-gbq/CONTRIBUTING.rst new file mode 100644 index 000000000000..3c7bb6217a44 --- /dev/null +++ b/packages/pandas-gbq/CONTRIBUTING.rst @@ -0,0 +1,277 @@ +.. Generated by synthtool. DO NOT EDIT! +############ +Contributing +############ + +#. **Please sign one of the contributor license agreements below.** +#. Fork the repo, develop and test your code changes, add docs. +#. Make sure that your commit messages clearly describe the changes. +#. Send a pull request. (Please Read: `Faster Pull Request Reviews`_) + +.. _Faster Pull Request Reviews: https://github.com/kubernetes/community/blob/master/contributors/guide/pull-requests.md#best-practices-for-faster-reviews + +.. contents:: Here are some guidelines for hacking on the Google Cloud Client libraries. + +*************** +Adding Features +*************** + +In order to add a feature: + +- The feature must be documented in both the API and narrative + documentation. + +- The feature must work fully on the following CPython versions: + 3.7, 3.8 and 3.9 on both UNIX and Windows. + +- The feature must not add unnecessary dependencies (where + "unnecessary" is of course subjective, but new dependencies should + be discussed). + +**************************** +Using a Development Checkout +**************************** + +You'll have to create a development environment using a Git checkout: + +- While logged into your GitHub account, navigate to the + ``python-bigquery-pandas`` `repo`_ on GitHub. + +- Fork and clone the ``python-bigquery-pandas`` repository to your GitHub account by + clicking the "Fork" button. + +- Clone your fork of ``python-bigquery-pandas`` from your GitHub account to your local + computer, substituting your account username and specifying the destination + as ``hack-on-python-bigquery-pandas``. E.g.:: + + $ cd ${HOME} + $ git clone git@github.com:USERNAME/python-bigquery-pandas.git hack-on-python-bigquery-pandas + $ cd hack-on-python-bigquery-pandas + # Configure remotes such that you can pull changes from the googleapis/python-bigquery-pandas + # repository into your local repository. + $ git remote add upstream git@github.com:googleapis/python-bigquery-pandas.git + # fetch and merge changes from upstream into main + $ git fetch upstream + $ git merge upstream/main + +Now your local repo is set up such that you will push changes to your GitHub +repo, from which you can submit a pull request. + +To work on the codebase and run the tests, we recommend using ``nox``, +but you can also use a ``virtualenv`` of your own creation. + +.. _repo: https://github.com/googleapis/python-bigquery-pandas + +Using ``nox`` +============= + +We use `nox `__ to instrument our tests. + +- To test your changes, run unit tests with ``nox``:: + $ nox -s unit + +- To run a single unit test:: + + $ nox -s unit-3.9 -- -k + + + .. note:: + + The unit tests and system tests are described in the + ``noxfile.py`` files in each directory. + +.. nox: https://pypi.org/project/nox/ + +***************************************** +I'm getting weird errors... Can you help? +***************************************** + +If the error mentions ``Python.h`` not being found, +install ``python-dev`` and try again. +On Debian/Ubuntu:: + + $ sudo apt-get install python-dev + +************ +Coding Style +************ +- We use the automatic code formatter ``black``. You can run it using + the nox session ``blacken``. This will eliminate many lint errors. Run via:: + + $ nox -s blacken + +- PEP8 compliance is required, with exceptions defined in the linter configuration. + If you have ``nox`` installed, you can test that you have not introduced + any non-compliant code via:: + + $ nox -s lint + +- In order to make ``nox -s lint`` run faster, you can set some environment + variables:: + + export GOOGLE_CLOUD_TESTING_REMOTE="upstream" + export GOOGLE_CLOUD_TESTING_BRANCH="main" + + By doing this, you are specifying the location of the most up-to-date + version of ``python-bigquery-pandas``. The + remote name ``upstream`` should point to the official ``googleapis`` + checkout and the branch should be the default branch on that remote (``main``). + +- This repository contains configuration for the + `pre-commit `__ tool, which automates checking + our linters during a commit. If you have it installed on your ``$PATH``, + you can enable enforcing those checks via: + +.. code-block:: bash + + $ pre-commit install + pre-commit installed at .git/hooks/pre-commit + +Exceptions to PEP8: + +- Many unit tests use a helper method, ``_call_fut`` ("FUT" is short for + "Function-Under-Test"), which is PEP8-incompliant, but more readable. + Some also use a local variable, ``MUT`` (short for "Module-Under-Test"). + +******************** +Running System Tests +******************** + +- To run system tests, you can execute:: + + # Run all system tests + $ nox -s system + + # Run a single system test + $ nox -s system-3.9 -- -k + + + .. note:: + + System tests are only configured to run under Python 3.8 and 3.9. + For expediency, we do not run them in older versions of Python 3. + + This alone will not run the tests. You'll need to change some local + auth settings and change some configuration in your project to + run all the tests. + +- System tests will be run against an actual project. You should use local credentials from gcloud when possible. See `Best practices for application authentication `__. Some tests require a service account. For those tests see `Authenticating as a service account `__. + +************* +Test Coverage +************* + +- The codebase *must* have 100% test statement coverage after each commit. + You can test coverage via ``nox -s cover``. + +****************************************************** +Documentation Coverage and Building HTML Documentation +****************************************************** + +If you fix a bug, and the bug requires an API or behavior modification, all +documentation in this package which references that API or behavior must be +changed to reflect the bug fix, ideally in the same commit that fixes the bug +or adds the feature. + +Build the docs via: + + $ nox -s docs + +************************* +Samples and code snippets +************************* + +Code samples and snippets live in the `samples/` catalogue. Feel free to +provide more examples, but make sure to write tests for those examples. +Each folder containing example code requires its own `noxfile.py` script +which automates testing. If you decide to create a new folder, you can +base it on the `samples/snippets` folder (providing `noxfile.py` and +the requirements files). + +The tests will run against a real Google Cloud Project, so you should +configure them just like the System Tests. + +- To run sample tests, you can execute:: + + # Run all tests in a folder + $ cd samples/snippets + $ nox -s py-3.8 + + # Run a single sample test + $ cd samples/snippets + $ nox -s py-3.8 -- -k + +******************************************** +Note About ``README`` as it pertains to PyPI +******************************************** + +The `description on PyPI`_ for the project comes directly from the +``README``. Due to the reStructuredText (``rst``) parser used by +PyPI, relative links which will work on GitHub (e.g. ``CONTRIBUTING.rst`` +instead of +``https://github.com/googleapis/python-bigquery-pandas/blob/main/CONTRIBUTING.rst``) +may cause problems creating links or rendering the description. + +.. _description on PyPI: https://pypi.org/project/pandas-gbq + + +************************* +Supported Python Versions +************************* + +We support: + +- `Python 3.7`_ +- `Python 3.8`_ +- `Python 3.9`_ + +.. _Python 3.7: https://docs.python.org/3.7/ +.. _Python 3.8: https://docs.python.org/3.8/ +.. _Python 3.9: https://docs.python.org/3.9/ + + +Supported versions can be found in our ``noxfile.py`` `config`_. + +.. _config: https://github.com/googleapis/python-bigquery-pandas/blob/main/noxfile.py + + +We also explicitly decided to support Python 3 beginning with version 3.7. +Reasons for this include: + +- Encouraging use of newest versions of Python 3 +- Taking the lead of `prominent`_ open-source `projects`_ +- `Unicode literal support`_ which allows for a cleaner codebase that + works in both Python 2 and Python 3 + +.. _prominent: https://docs.djangoproject.com/en/1.9/faq/install/#what-python-version-can-i-use-with-django +.. _projects: http://flask.pocoo.org/docs/0.10/python3/ +.. _Unicode literal support: https://www.python.org/dev/peps/pep-0414/ + +********** +Versioning +********** + +This library follows `Semantic Versioning`_. + +.. _Semantic Versioning: http://semver.org/ + +Some packages are currently in major version zero (``0.y.z``), which means that +anything may change at any time and the public API should not be considered +stable. + +****************************** +Contributor License Agreements +****************************** + +Before we can accept your pull requests you'll need to sign a Contributor +License Agreement (CLA): + +- **If you are an individual writing original source code** and **you own the + intellectual property**, then you'll need to sign an + `individual CLA `__. +- **If you work for a company that wants to allow you to contribute your work**, + then you'll need to sign a + `corporate CLA `__. + +You can sign these electronically (just scroll to the bottom). After that, +we'll be able to accept your pull requests. diff --git a/packages/pandas-gbq/MANIFEST.in b/packages/pandas-gbq/MANIFEST.in index c27fd5f3d581..e783f4c6209b 100644 --- a/packages/pandas-gbq/MANIFEST.in +++ b/packages/pandas-gbq/MANIFEST.in @@ -1,19 +1,25 @@ -include MANIFEST.in -include README.rst -include AUTHORS.md -include LICENSE.txt -include setup.py +# -*- coding: utf-8 -*- +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -graft pandas_gbq +# Generated by synthtool. DO NOT EDIT! +include README.rst LICENSE +recursive-include google *.json *.proto py.typed +recursive-include tests * +global-exclude *.py[co] +global-exclude __pycache__ -global-exclude *.so -global-exclude *.pyd -global-exclude *.pyc -global-exclude *~ -global-exclude \#* -global-exclude .git* -global-exclude .DS_Store -global-exclude *.png - -include versioneer.py -include pandas_gbq/_version.py +# Exclude scripts for samples readmegen +prune scripts/readme-gen diff --git a/packages/pandas-gbq/ci/run_tests.sh b/packages/pandas-gbq/ci/run_tests.sh index efc12fe148a1..8a1d7f911355 100755 --- a/packages/pandas-gbq/ci/run_tests.sh +++ b/packages/pandas-gbq/ci/run_tests.sh @@ -12,4 +12,4 @@ fi # Install test requirements pip install coverage pytest pytest-cov flake8 codecov google-cloud-testutils -pytest -v -m "not local_auth" --cov=pandas_gbq --cov-report xml:/tmp/pytest-cov.xml tests +pytest -v -m "not local_auth" --cov=pandas_gbq --cov-report xml:/tmp/pytest-cov.xml --cov-fail-under=0 tests diff --git a/packages/pandas-gbq/docs/_static/custom.css b/packages/pandas-gbq/docs/_static/custom.css new file mode 100644 index 000000000000..b0a295464b23 --- /dev/null +++ b/packages/pandas-gbq/docs/_static/custom.css @@ -0,0 +1,20 @@ +div#python2-eol { + border-color: red; + border-width: medium; +} + +/* Ensure minimum width for 'Parameters' / 'Returns' column */ +dl.field-list > dt { + min-width: 100px +} + +/* Insert space between methods for readability */ +dl.method { + padding-top: 10px; + padding-bottom: 10px +} + +/* Insert empty space between classes */ +dl.class { + padding-bottom: 50px +} diff --git a/packages/pandas-gbq/docs/_templates/layout.html b/packages/pandas-gbq/docs/_templates/layout.html index 74f6910cf8fe..6316a537f72b 100644 --- a/packages/pandas-gbq/docs/_templates/layout.html +++ b/packages/pandas-gbq/docs/_templates/layout.html @@ -1,8 +1,50 @@ - {% extends "!layout.html" %} -{% set css_files = css_files + ["_static/style.css"] %} +{%- block content %} +{%- if theme_fixed_sidebar|lower == 'true' %} +

+ {{ sidebar() }} + {%- block document %} +
+ {%- if render_sidebar %} +
+ {%- endif %} + + {%- block relbar_top %} + {%- if theme_show_relbar_top|tobool %} +
+   + {{- rellink_markup () }} +
+ {%- endif %} + {% endblock %} + +
+
+ As of January 1, 2020 this library no longer supports Python 2 on the latest released version. + Library versions released prior to that date will continue to be available. For more information please + visit Python 2 support on Google Cloud. +
+ {% block body %} {% endblock %} +
+ + {%- block relbar_bottom %} + {%- if theme_show_relbar_bottom|tobool %} +
+   + {{- rellink_markup () }} +
+ {%- endif %} + {% endblock %} + + {%- if render_sidebar %} +
+ {%- endif %} +
+ {%- endblock %} +
+
+{%- else %} +{{ super() }} +{%- endif %} +{%- endblock %} diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index da42b7461800..2542369dabec 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -175,7 +175,7 @@ def cover(session): test runs (not system test runs), and then erases coverage data. """ session.install("coverage", "pytest-cov") - session.run("coverage", "report", "--show-missing", "--fail-under=73") + session.run("coverage", "report", "--show-missing", "--fail-under=86") session.run("coverage", "erase") diff --git a/packages/pandas-gbq/owlbot.py b/packages/pandas-gbq/owlbot.py new file mode 100644 index 000000000000..960dca479028 --- /dev/null +++ b/packages/pandas-gbq/owlbot.py @@ -0,0 +1,82 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This script is used to synthesize generated parts of this library.""" + +import pathlib + +import synthtool as s +from synthtool import gcp +from synthtool.languages import python + +REPO_ROOT = pathlib.Path(__file__).parent.absolute() + +common = gcp.CommonTemplates() + +# ---------------------------------------------------------------------------- +# Add templated files +# ---------------------------------------------------------------------------- + +extras = ["tqdm"] +templated_files = common.py_library( + unit_test_python_versions=["3.7", "3.8", "3.9"], + system_test_python_versions=["3.8", "3.9"], + cov_level=86, + unit_test_extras=extras, + system_test_extras=extras, + intersphinx_dependencies={ + "pandas": "https://pandas.pydata.org/pandas-docs/stable/", + "pydata-google-auth": "https://pydata-google-auth.readthedocs.io/en/latest/", + }, +) +s.move( + templated_files, + excludes=[ + # pandas-gbq was originally licensed BSD-3-Clause License + "LICENSE", + # Mulit-processing note isn't relevant, as pandas_gbq is responsible for + # creating clients, not the end user. + "docs/multiprocessing.rst", + ], +) + +# ---------------------------------------------------------------------------- +# Fixup files +# ---------------------------------------------------------------------------- + +s.replace( + ["noxfile.py"], r"[\"']google[\"']", '"pandas_gbq"', +) + +s.replace( + ["noxfile.py"], "google/cloud", "pandas_gbq", +) + +s.replace( + [".github/header-checker-lint.yml"], '"Google LLC"', '"pandas-gbq Authors"', +) + +# ---------------------------------------------------------------------------- +# Samples templates +# ---------------------------------------------------------------------------- + +python.py_samples(skip_readmes=True) + +# ---------------------------------------------------------------------------- +# Final cleanup +# ---------------------------------------------------------------------------- + +s.shell.run(["nox", "-s", "blacken"], hide_output=False) +for noxfile in REPO_ROOT.glob("samples/**/noxfile.py"): + s.shell.run(["nox", "-s", "blacken"], cwd=noxfile.parent, hide_output=False) diff --git a/packages/pandas-gbq/renovate.json b/packages/pandas-gbq/renovate.json index f45d8f110c30..c21036d385e5 100644 --- a/packages/pandas-gbq/renovate.json +++ b/packages/pandas-gbq/renovate.json @@ -1,5 +1,12 @@ { "extends": [ - "config:base" - ] + "config:base", + "group:all", + ":preserveSemverRanges", + ":disableDependencyDashboard" + ], + "ignorePaths": [".pre-commit-config.yaml"], + "pip_requirements": { + "fileMatch": ["requirements-test.txt", "samples/[\\S/]*constraints.txt", "samples/[\\S/]*constraints-test.txt"] + } } diff --git a/packages/pandas-gbq/samples/snippets/conftest.py b/packages/pandas-gbq/samples/snippets/conftest.py index e5a3a7d9c174..0d0ae0916618 100644 --- a/packages/pandas-gbq/samples/snippets/conftest.py +++ b/packages/pandas-gbq/samples/snippets/conftest.py @@ -15,7 +15,7 @@ def cleanup_datasets(bigquery_client: bigquery.Client): for dataset in bigquery_client.list_datasets(): if prefixer.should_cleanup(dataset.dataset_id): bigquery_client.delete_dataset( - dataset, delete_contents=True, not_found_ok=True + dataset.reference, delete_contents=True, not_found_ok=True ) diff --git a/packages/pandas-gbq/samples/snippets/noxfile.py b/packages/pandas-gbq/samples/snippets/noxfile.py index a57b633c0cb7..b008613f03ff 100644 --- a/packages/pandas-gbq/samples/snippets/noxfile.py +++ b/packages/pandas-gbq/samples/snippets/noxfile.py @@ -87,7 +87,7 @@ def get_pytest_env_vars() -> Dict[str, str]: # DO NOT EDIT - automatically generated. # All versions used to test samples. -ALL_VERSIONS = ["3.7", "3.8", "3.9"] +ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9"] # Any default versions that should be ignored. IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] diff --git a/packages/pandas-gbq/samples/snippets/noxfile_config.py b/packages/pandas-gbq/samples/snippets/noxfile_config.py new file mode 100644 index 000000000000..2bceb2fe12fe --- /dev/null +++ b/packages/pandas-gbq/samples/snippets/noxfile_config.py @@ -0,0 +1,7 @@ +# Copyright (c) 2021 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +TEST_CONFIG_OVERRIDE = { + "ignored_versions": ["2.7", "3.6"], +} diff --git a/packages/pandas-gbq/scripts/decrypt-secrets.sh b/packages/pandas-gbq/scripts/decrypt-secrets.sh new file mode 100755 index 000000000000..21f6d2a26d90 --- /dev/null +++ b/packages/pandas-gbq/scripts/decrypt-secrets.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +ROOT=$( dirname "$DIR" ) + +# Work from the project root. +cd $ROOT + +# Prevent it from overriding files. +# We recommend that sample authors use their own service account files and cloud project. +# In that case, they are supposed to prepare these files by themselves. +if [[ -f "testing/test-env.sh" ]] || \ + [[ -f "testing/service-account.json" ]] || \ + [[ -f "testing/client-secrets.json" ]]; then + echo "One or more target files exist, aborting." + exit 1 +fi + +# Use SECRET_MANAGER_PROJECT if set, fallback to cloud-devrel-kokoro-resources. +PROJECT_ID="${SECRET_MANAGER_PROJECT:-cloud-devrel-kokoro-resources}" + +gcloud secrets versions access latest --secret="python-docs-samples-test-env" \ + --project="${PROJECT_ID}" \ + > testing/test-env.sh +gcloud secrets versions access latest \ + --secret="python-docs-samples-service-account" \ + --project="${PROJECT_ID}" \ + > testing/service-account.json +gcloud secrets versions access latest \ + --secret="python-docs-samples-client-secrets" \ + --project="${PROJECT_ID}" \ + > testing/client-secrets.json diff --git a/packages/pandas-gbq/scripts/readme-gen/readme_gen.py b/packages/pandas-gbq/scripts/readme-gen/readme_gen.py new file mode 100644 index 000000000000..d309d6e97518 --- /dev/null +++ b/packages/pandas-gbq/scripts/readme-gen/readme_gen.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generates READMEs using configuration defined in yaml.""" + +import argparse +import io +import os +import subprocess + +import jinja2 +import yaml + + +jinja_env = jinja2.Environment( + trim_blocks=True, + loader=jinja2.FileSystemLoader( + os.path.abspath(os.path.join(os.path.dirname(__file__), 'templates')))) + +README_TMPL = jinja_env.get_template('README.tmpl.rst') + + +def get_help(file): + return subprocess.check_output(['python', file, '--help']).decode() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('source') + parser.add_argument('--destination', default='README.rst') + + args = parser.parse_args() + + source = os.path.abspath(args.source) + root = os.path.dirname(source) + destination = os.path.join(root, args.destination) + + jinja_env.globals['get_help'] = get_help + + with io.open(source, 'r') as f: + config = yaml.load(f) + + # This allows get_help to execute in the right directory. + os.chdir(root) + + output = README_TMPL.render(config) + + with io.open(destination, 'w') as f: + f.write(output) + + +if __name__ == '__main__': + main() diff --git a/packages/pandas-gbq/scripts/readme-gen/templates/README.tmpl.rst b/packages/pandas-gbq/scripts/readme-gen/templates/README.tmpl.rst new file mode 100644 index 000000000000..4fd239765b0a --- /dev/null +++ b/packages/pandas-gbq/scripts/readme-gen/templates/README.tmpl.rst @@ -0,0 +1,87 @@ +{# The following line is a lie. BUT! Once jinja2 is done with it, it will + become truth! #} +.. This file is automatically generated. Do not edit this file directly. + +{{product.name}} Python Samples +=============================================================================== + +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor={{folder}}/README.rst + + +This directory contains samples for {{product.name}}. {{product.description}} + +{{description}} + +.. _{{product.name}}: {{product.url}} + +{% if required_api_url %} +To run the sample, you need to enable the API at: {{required_api_url}} +{% endif %} + +{% if required_role %} +To run the sample, you need to have `{{required_role}}` role. +{% endif %} + +{{other_required_steps}} + +{% if setup %} +Setup +------------------------------------------------------------------------------- + +{% for section in setup %} + +{% include section + '.tmpl.rst' %} + +{% endfor %} +{% endif %} + +{% if samples %} +Samples +------------------------------------------------------------------------------- + +{% for sample in samples %} +{{sample.name}} ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +{% if not sample.hide_cloudshell_button %} +.. image:: https://gstatic.com/cloudssh/images/open-btn.png + :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor={{folder}}/{{sample.file}},{{folder}}/README.rst +{% endif %} + + +{{sample.description}} + +To run this sample: + +.. code-block:: bash + + $ python {{sample.file}} +{% if sample.show_help %} + + {{get_help(sample.file)|indent}} +{% endif %} + + +{% endfor %} +{% endif %} + +{% if cloud_client_library %} + +The client library +------------------------------------------------------------------------------- + +This sample uses the `Google Cloud Client Library for Python`_. +You can read the documentation for more details on API usage and use GitHub +to `browse the source`_ and `report issues`_. + +.. _Google Cloud Client Library for Python: + https://googlecloudplatform.github.io/google-cloud-python/ +.. _browse the source: + https://github.com/GoogleCloudPlatform/google-cloud-python +.. _report issues: + https://github.com/GoogleCloudPlatform/google-cloud-python/issues + +{% endif %} + +.. _Google Cloud SDK: https://cloud.google.com/sdk/ \ No newline at end of file diff --git a/packages/pandas-gbq/scripts/readme-gen/templates/auth.tmpl.rst b/packages/pandas-gbq/scripts/readme-gen/templates/auth.tmpl.rst new file mode 100644 index 000000000000..1446b94a5e3a --- /dev/null +++ b/packages/pandas-gbq/scripts/readme-gen/templates/auth.tmpl.rst @@ -0,0 +1,9 @@ +Authentication +++++++++++++++ + +This sample requires you to have authentication setup. Refer to the +`Authentication Getting Started Guide`_ for instructions on setting up +credentials for applications. + +.. _Authentication Getting Started Guide: + https://cloud.google.com/docs/authentication/getting-started diff --git a/packages/pandas-gbq/scripts/readme-gen/templates/auth_api_key.tmpl.rst b/packages/pandas-gbq/scripts/readme-gen/templates/auth_api_key.tmpl.rst new file mode 100644 index 000000000000..11957ce2714a --- /dev/null +++ b/packages/pandas-gbq/scripts/readme-gen/templates/auth_api_key.tmpl.rst @@ -0,0 +1,14 @@ +Authentication +++++++++++++++ + +Authentication for this service is done via an `API Key`_. To obtain an API +Key: + +1. Open the `Cloud Platform Console`_ +2. Make sure that billing is enabled for your project. +3. From the **Credentials** page, create a new **API Key** or use an existing + one for your project. + +.. _API Key: + https://developers.google.com/api-client-library/python/guide/aaa_apikeys +.. _Cloud Console: https://console.cloud.google.com/project?_ diff --git a/packages/pandas-gbq/scripts/readme-gen/templates/install_deps.tmpl.rst b/packages/pandas-gbq/scripts/readme-gen/templates/install_deps.tmpl.rst new file mode 100644 index 000000000000..275d649890d7 --- /dev/null +++ b/packages/pandas-gbq/scripts/readme-gen/templates/install_deps.tmpl.rst @@ -0,0 +1,29 @@ +Install Dependencies +++++++++++++++++++++ + +#. Clone python-docs-samples and change directory to the sample directory you want to use. + + .. code-block:: bash + + $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + +#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. + + .. _Python Development Environment Setup Guide: + https://cloud.google.com/python/setup + +#. Create a virtualenv. Samples are compatible with Python 3.6+. + + .. code-block:: bash + + $ virtualenv env + $ source env/bin/activate + +#. Install the dependencies needed to run the samples. + + .. code-block:: bash + + $ pip install -r requirements.txt + +.. _pip: https://pip.pypa.io/ +.. _virtualenv: https://virtualenv.pypa.io/ diff --git a/packages/pandas-gbq/scripts/readme-gen/templates/install_portaudio.tmpl.rst b/packages/pandas-gbq/scripts/readme-gen/templates/install_portaudio.tmpl.rst new file mode 100644 index 000000000000..5ea33d18c00c --- /dev/null +++ b/packages/pandas-gbq/scripts/readme-gen/templates/install_portaudio.tmpl.rst @@ -0,0 +1,35 @@ +Install PortAudio ++++++++++++++++++ + +Install `PortAudio`_. This is required by the `PyAudio`_ library to stream +audio from your computer's microphone. PyAudio depends on PortAudio for cross-platform compatibility, and is installed differently depending on the +platform. + +* For Mac OS X, you can use `Homebrew`_:: + + brew install portaudio + + **Note**: if you encounter an error when running `pip install` that indicates + it can't find `portaudio.h`, try running `pip install` with the following + flags:: + + pip install --global-option='build_ext' \ + --global-option='-I/usr/local/include' \ + --global-option='-L/usr/local/lib' \ + pyaudio + +* For Debian / Ubuntu Linux:: + + apt-get install portaudio19-dev python-all-dev + +* Windows may work without having to install PortAudio explicitly (it will get + installed with PyAudio). + +For more details, see the `PyAudio installation`_ page. + + +.. _PyAudio: https://people.csail.mit.edu/hubert/pyaudio/ +.. _PortAudio: http://www.portaudio.com/ +.. _PyAudio installation: + https://people.csail.mit.edu/hubert/pyaudio/#downloads +.. _Homebrew: http://brew.sh diff --git a/packages/pandas-gbq/testing/.gitignore b/packages/pandas-gbq/testing/.gitignore new file mode 100644 index 000000000000..b05fbd630881 --- /dev/null +++ b/packages/pandas-gbq/testing/.gitignore @@ -0,0 +1,3 @@ +test-env.sh +service-account.json +client-secrets.json \ No newline at end of file diff --git a/packages/pandas-gbq/tests/system/conftest.py b/packages/pandas-gbq/tests/system/conftest.py index c29649029b80..6ac55220262a 100644 --- a/packages/pandas-gbq/tests/system/conftest.py +++ b/packages/pandas-gbq/tests/system/conftest.py @@ -31,7 +31,7 @@ def cleanup_datasets(bigquery_client: bigquery.Client): for dataset in bigquery_client.list_datasets(): if prefixer.should_cleanup(dataset.dataset_id): bigquery_client.delete_dataset( - dataset, delete_contents=True, not_found_ok=True + dataset.reference, delete_contents=True, not_found_ok=True ) From 89ceb8e95fc27643c6cb78c720216558b6ea6125 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 5 Oct 2021 13:46:26 -0600 Subject: [PATCH 235/519] build: use trampoline_v2 for python samples and allow custom dockerfile (#404) Source-Link: https://github.com/googleapis/synthtool/commit/a7ed11ec0863c422ba2e73aafa75eab22c32b33d Post-Processor: gcr.io/repo-automation-bots/owlbot-python:latest@sha256:effb79ef0525b02611cada94df8d7a248758928579c03d522f26ed37f9867668 Co-authored-by: Owl Bot --- packages/pandas-gbq/.coveragerc | 2 +- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 ++-- .../pandas-gbq/.kokoro/samples/lint/common.cfg | 2 +- .../.kokoro/samples/python3.6/common.cfg | 2 +- .../.kokoro/samples/python3.6/periodic.cfg | 2 +- .../.kokoro/samples/python3.7/common.cfg | 2 +- .../.kokoro/samples/python3.7/periodic.cfg | 2 +- .../.kokoro/samples/python3.8/common.cfg | 2 +- .../.kokoro/samples/python3.8/periodic.cfg | 2 +- .../.kokoro/samples/python3.9/common.cfg | 2 +- .../.kokoro/samples/python3.9/periodic.cfg | 2 +- .../.kokoro/test-samples-against-head.sh | 2 -- packages/pandas-gbq/.kokoro/test-samples.sh | 2 -- packages/pandas-gbq/.trampolinerc | 17 ++++++++++++++--- packages/pandas-gbq/samples/snippets/noxfile.py | 4 ++++ 15 files changed, 30 insertions(+), 19 deletions(-) diff --git a/packages/pandas-gbq/.coveragerc b/packages/pandas-gbq/.coveragerc index 0d8e6297dc9c..d62303d636ee 100644 --- a/packages/pandas-gbq/.coveragerc +++ b/packages/pandas-gbq/.coveragerc @@ -21,7 +21,7 @@ omit = google/cloud/__init__.py [report] -fail_under = 100 +fail_under = 86 show_missing = True exclude_lines = # Re-enable the standard pragma diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 7b6cc31057ef..e24edd9af228 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: - image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:a3a85c2e0b3293068e47b1635b178f7e3d3845f2cfb8722de6713d4bbafdcd1d + image: gcr.io/repo-automation-bots/owlbot-python:latest + digest: sha256:effb79ef0525b02611cada94df8d7a248758928579c03d522f26ed37f9867668 diff --git a/packages/pandas-gbq/.kokoro/samples/lint/common.cfg b/packages/pandas-gbq/.kokoro/samples/lint/common.cfg index 9e6f650c1dc0..80ead70e5521 100644 --- a/packages/pandas-gbq/.kokoro/samples/lint/common.cfg +++ b/packages/pandas-gbq/.kokoro/samples/lint/common.cfg @@ -31,4 +31,4 @@ gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" # Use the trampoline script to run in docker. -build_file: "python-bigquery-pandas/.kokoro/trampoline.sh" \ No newline at end of file +build_file: "python-bigquery-pandas/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.6/common.cfg b/packages/pandas-gbq/.kokoro/samples/python3.6/common.cfg index ced3d18c11c8..bf7686db795a 100644 --- a/packages/pandas-gbq/.kokoro/samples/python3.6/common.cfg +++ b/packages/pandas-gbq/.kokoro/samples/python3.6/common.cfg @@ -37,4 +37,4 @@ gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" # Use the trampoline script to run in docker. -build_file: "python-bigquery-pandas/.kokoro/trampoline.sh" \ No newline at end of file +build_file: "python-bigquery-pandas/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.6/periodic.cfg b/packages/pandas-gbq/.kokoro/samples/python3.6/periodic.cfg index 50fec9649732..71cd1e597e38 100644 --- a/packages/pandas-gbq/.kokoro/samples/python3.6/periodic.cfg +++ b/packages/pandas-gbq/.kokoro/samples/python3.6/periodic.cfg @@ -3,4 +3,4 @@ env_vars: { key: "INSTALL_LIBRARY_FROM_SOURCE" value: "False" -} \ No newline at end of file +} diff --git a/packages/pandas-gbq/.kokoro/samples/python3.7/common.cfg b/packages/pandas-gbq/.kokoro/samples/python3.7/common.cfg index a61c609671d7..be202f330704 100644 --- a/packages/pandas-gbq/.kokoro/samples/python3.7/common.cfg +++ b/packages/pandas-gbq/.kokoro/samples/python3.7/common.cfg @@ -37,4 +37,4 @@ gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" # Use the trampoline script to run in docker. -build_file: "python-bigquery-pandas/.kokoro/trampoline.sh" \ No newline at end of file +build_file: "python-bigquery-pandas/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.7/periodic.cfg b/packages/pandas-gbq/.kokoro/samples/python3.7/periodic.cfg index 50fec9649732..71cd1e597e38 100644 --- a/packages/pandas-gbq/.kokoro/samples/python3.7/periodic.cfg +++ b/packages/pandas-gbq/.kokoro/samples/python3.7/periodic.cfg @@ -3,4 +3,4 @@ env_vars: { key: "INSTALL_LIBRARY_FROM_SOURCE" value: "False" -} \ No newline at end of file +} diff --git a/packages/pandas-gbq/.kokoro/samples/python3.8/common.cfg b/packages/pandas-gbq/.kokoro/samples/python3.8/common.cfg index 75983c570b49..7424a3b9d52b 100644 --- a/packages/pandas-gbq/.kokoro/samples/python3.8/common.cfg +++ b/packages/pandas-gbq/.kokoro/samples/python3.8/common.cfg @@ -37,4 +37,4 @@ gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" # Use the trampoline script to run in docker. -build_file: "python-bigquery-pandas/.kokoro/trampoline.sh" \ No newline at end of file +build_file: "python-bigquery-pandas/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.8/periodic.cfg b/packages/pandas-gbq/.kokoro/samples/python3.8/periodic.cfg index 50fec9649732..71cd1e597e38 100644 --- a/packages/pandas-gbq/.kokoro/samples/python3.8/periodic.cfg +++ b/packages/pandas-gbq/.kokoro/samples/python3.8/periodic.cfg @@ -3,4 +3,4 @@ env_vars: { key: "INSTALL_LIBRARY_FROM_SOURCE" value: "False" -} \ No newline at end of file +} diff --git a/packages/pandas-gbq/.kokoro/samples/python3.9/common.cfg b/packages/pandas-gbq/.kokoro/samples/python3.9/common.cfg index 0e88731298a0..c0948d2112b0 100644 --- a/packages/pandas-gbq/.kokoro/samples/python3.9/common.cfg +++ b/packages/pandas-gbq/.kokoro/samples/python3.9/common.cfg @@ -37,4 +37,4 @@ gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" # Use the trampoline script to run in docker. -build_file: "python-bigquery-pandas/.kokoro/trampoline.sh" \ No newline at end of file +build_file: "python-bigquery-pandas/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.9/periodic.cfg b/packages/pandas-gbq/.kokoro/samples/python3.9/periodic.cfg index 50fec9649732..71cd1e597e38 100644 --- a/packages/pandas-gbq/.kokoro/samples/python3.9/periodic.cfg +++ b/packages/pandas-gbq/.kokoro/samples/python3.9/periodic.cfg @@ -3,4 +3,4 @@ env_vars: { key: "INSTALL_LIBRARY_FROM_SOURCE" value: "False" -} \ No newline at end of file +} diff --git a/packages/pandas-gbq/.kokoro/test-samples-against-head.sh b/packages/pandas-gbq/.kokoro/test-samples-against-head.sh index b382b13f8a88..ba3a707b040c 100755 --- a/packages/pandas-gbq/.kokoro/test-samples-against-head.sh +++ b/packages/pandas-gbq/.kokoro/test-samples-against-head.sh @@ -23,6 +23,4 @@ set -eo pipefail # Enables `**` to include files nested inside sub-folders shopt -s globstar -cd github/python-bigquery-pandas - exec .kokoro/test-samples-impl.sh diff --git a/packages/pandas-gbq/.kokoro/test-samples.sh b/packages/pandas-gbq/.kokoro/test-samples.sh index 9342d9522fa8..11c042d342d7 100755 --- a/packages/pandas-gbq/.kokoro/test-samples.sh +++ b/packages/pandas-gbq/.kokoro/test-samples.sh @@ -24,8 +24,6 @@ set -eo pipefail # Enables `**` to include files nested inside sub-folders shopt -s globstar -cd github/python-bigquery-pandas - # Run periodic samples tests at latest release if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then # preserving the test runner implementation. diff --git a/packages/pandas-gbq/.trampolinerc b/packages/pandas-gbq/.trampolinerc index 383b6ec89fbc..0eee72ab62aa 100644 --- a/packages/pandas-gbq/.trampolinerc +++ b/packages/pandas-gbq/.trampolinerc @@ -16,15 +16,26 @@ # Add required env vars here. required_envvars+=( - "STAGING_BUCKET" - "V2_STAGING_BUCKET" ) # Add env vars which are passed down into the container here. pass_down_envvars+=( + "NOX_SESSION" + ############### + # Docs builds + ############### "STAGING_BUCKET" "V2_STAGING_BUCKET" - "NOX_SESSION" + ################## + # Samples builds + ################## + "INSTALL_LIBRARY_FROM_SOURCE" + "RUN_TESTS_SESSION" + "BUILD_SPECIFIC_GCLOUD_PROJECT" + # Target directories. + "RUN_TESTS_DIRS" + # The nox session to run. + "RUN_TESTS_SESSION" ) # Prevent unintentional override on the default image. diff --git a/packages/pandas-gbq/samples/snippets/noxfile.py b/packages/pandas-gbq/samples/snippets/noxfile.py index b008613f03ff..1fd8956fbf01 100644 --- a/packages/pandas-gbq/samples/snippets/noxfile.py +++ b/packages/pandas-gbq/samples/snippets/noxfile.py @@ -98,6 +98,10 @@ def get_pytest_env_vars() -> Dict[str, str]: "True", "true", ) + +# Error if a python version is missing +nox.options.error_on_missing_interpreters = True + # # Style Checks # From 3e76a699fb0de43861616716ca42475ba3f54210 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 12 Oct 2021 16:44:42 -0400 Subject: [PATCH 236/519] chore(python): fix formatting issue in noxfile.py.j2 (#406) Source-Link: https://github.com/googleapis/synthtool/commit/0e85ed6ccf43fb433c03551205c9a186a2da1d4c Post-Processor: gcr.io/repo-automation-bots/owlbot-python:latest@sha256:06caa6588d5916680e9b26de7bcd20980ef9b6abd8bab077d00e1f75c4e34f2f Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 2 +- .../.kokoro/samples/python3.10/common.cfg | 40 +++++++++++++++++++ .../.kokoro/samples/python3.10/continuous.cfg | 6 +++ .../samples/python3.10/periodic-head.cfg | 11 +++++ .../.kokoro/samples/python3.10/periodic.cfg | 6 +++ .../.kokoro/samples/python3.10/presubmit.cfg | 6 +++ .../pandas-gbq/samples/snippets/noxfile.py | 2 +- 7 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.10/common.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.10/continuous.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.10/periodic-head.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.10/periodic.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.10/presubmit.cfg diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index e24edd9af228..b4163c02477e 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/repo-automation-bots/owlbot-python:latest - digest: sha256:effb79ef0525b02611cada94df8d7a248758928579c03d522f26ed37f9867668 + digest: sha256:06caa6588d5916680e9b26de7bcd20980ef9b6abd8bab077d00e1f75c4e34f2f diff --git a/packages/pandas-gbq/.kokoro/samples/python3.10/common.cfg b/packages/pandas-gbq/.kokoro/samples/python3.10/common.cfg new file mode 100644 index 000000000000..3c9e744f27e2 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.10/common.cfg @@ -0,0 +1,40 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.10" +} + +# Declare build specific Cloud project. +env_vars: { + key: "BUILD_SPECIFIC_GCLOUD_PROJECT" + value: "python-docs-samples-tests-310" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-pandas/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-bigquery-pandas/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.10/continuous.cfg b/packages/pandas-gbq/.kokoro/samples/python3.10/continuous.cfg new file mode 100644 index 000000000000..a1c8d9759c88 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.10/continuous.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.10/periodic-head.cfg b/packages/pandas-gbq/.kokoro/samples/python3.10/periodic-head.cfg new file mode 100644 index 000000000000..98efde4dc99d --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.10/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-pandas/.kokoro/test-samples-against-head.sh" +} diff --git a/packages/pandas-gbq/.kokoro/samples/python3.10/periodic.cfg b/packages/pandas-gbq/.kokoro/samples/python3.10/periodic.cfg new file mode 100644 index 000000000000..71cd1e597e38 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.10/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} diff --git a/packages/pandas-gbq/.kokoro/samples/python3.10/presubmit.cfg b/packages/pandas-gbq/.kokoro/samples/python3.10/presubmit.cfg new file mode 100644 index 000000000000..a1c8d9759c88 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.10/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/packages/pandas-gbq/samples/snippets/noxfile.py b/packages/pandas-gbq/samples/snippets/noxfile.py index 1fd8956fbf01..93a9122cc457 100644 --- a/packages/pandas-gbq/samples/snippets/noxfile.py +++ b/packages/pandas-gbq/samples/snippets/noxfile.py @@ -87,7 +87,7 @@ def get_pytest_env_vars() -> Dict[str, str]: # DO NOT EDIT - automatically generated. # All versions used to test samples. -ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9"] +ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10"] # Any default versions that should be ignored. IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] From 895984fda2a33ee0f6327be62b0b5f6c5968f1ab Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 21 Oct 2021 19:29:47 -0400 Subject: [PATCH 237/519] chore(python): use post processor image in cloud-devrel-public-resources (#410) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(python): modify templated noxfile to support non-cloud APIs Source-Link: https://github.com/googleapis/synthtool/commit/76d5fec7a9e77a12c28654b333103578623a0c1b Post-Processor: gcr.io/repo-automation-bots/owlbot-python:latest@sha256:0e17f66ec39d87a7e64954d7bf254dc2d05347f5aefbb3a1d4a3270fc7d6ea97 * fix replacement in owlbot.py * use post processor image in cloud-devrel-public-resources * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * update replacement in owlbot.py Co-authored-by: Owl Bot Co-authored-by: Anthonios Partheniou --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 ++-- packages/pandas-gbq/.github/.OwlBot.yaml | 2 +- packages/pandas-gbq/owlbot.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index b4163c02477e..4423944431a1 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: - image: gcr.io/repo-automation-bots/owlbot-python:latest - digest: sha256:06caa6588d5916680e9b26de7bcd20980ef9b6abd8bab077d00e1f75c4e34f2f + image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest + digest: sha256:979d9498e07c50097c1aeda937dcd32094ecc7440278a83e832b6a05602f62b6 diff --git a/packages/pandas-gbq/.github/.OwlBot.yaml b/packages/pandas-gbq/.github/.OwlBot.yaml index 8642b5f3eaa4..33779d65e446 100644 --- a/packages/pandas-gbq/.github/.OwlBot.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.yaml @@ -13,7 +13,7 @@ # limitations under the License. docker: - image: gcr.io/repo-automation-bots/owlbot-python:latest + image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest begin-after-commit-hash: 1afeb53252641dc35a421fa5acc59e2f3229ad6d diff --git a/packages/pandas-gbq/owlbot.py b/packages/pandas-gbq/owlbot.py index 960dca479028..d382cf6658e3 100644 --- a/packages/pandas-gbq/owlbot.py +++ b/packages/pandas-gbq/owlbot.py @@ -60,7 +60,7 @@ ) s.replace( - ["noxfile.py"], "google/cloud", "pandas_gbq", + ["noxfile.py"], "--cov=google", "--cov=pandas_gbq", ) s.replace( From 8248975a074c1c00d494f40b45d55be80c64e39f Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 25 Oct 2021 16:14:16 -0500 Subject: [PATCH 238/519] test: correct policy tags REST representation (#388) There need to be some changes in `google-cloud-bigquery` for this fix: https://github.com/googleapis/python-bigquery/pull/703#issuecomment-925107546 (Update: Done!) Closes #387 --- packages/pandas-gbq/tests/unit/test_load.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/tests/unit/test_load.py b/packages/pandas-gbq/tests/unit/test_load.py index d00495a6548e..2ddb4f50c84d 100644 --- a/packages/pandas-gbq/tests/unit/test_load.py +++ b/packages/pandas-gbq/tests/unit/test_load.py @@ -112,7 +112,9 @@ def test_load_chunks_omits_policy_tags( "my-project.my_dataset.my_table" ) schema = { - "fields": [{"name": "col1", "type": "INT64", "policyTags": ["tag1", "tag2"]}] + "fields": [ + {"name": "col1", "type": "INT64", "policyTags": {"names": ["tag1", "tag2"]}} + ] } _ = list(load.load_chunks(mock_bigquery_client, df, destination, schema=schema)) From b0badf466b3bab9626068f23d5106a62ea406d60 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Mon, 25 Oct 2021 21:44:03 -0400 Subject: [PATCH 239/519] chore(python): block pushing non-cloud docs to Cloud RAD (#412) Source-Link: https://github.com/googleapis/synthtool/commit/694118b039b09551fb5d445fceb361a7dbb06400 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:ec49167c606648a063d1222220b48119c912562849a0528f35bfb592a9f72737 Co-authored-by: Owl Bot --- packages/pandas-gbq/.coveragerc | 1 + packages/pandas-gbq/.github/.OwlBot.lock.yaml | 2 +- packages/pandas-gbq/.kokoro/docs/common.cfg | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/.coveragerc b/packages/pandas-gbq/.coveragerc index d62303d636ee..61285af5d8f2 100644 --- a/packages/pandas-gbq/.coveragerc +++ b/packages/pandas-gbq/.coveragerc @@ -18,6 +18,7 @@ [run] branch = True omit = + google/__init__.py google/cloud/__init__.py [report] diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 4423944431a1..cb89b2e326b7 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:979d9498e07c50097c1aeda937dcd32094ecc7440278a83e832b6a05602f62b6 + digest: sha256:ec49167c606648a063d1222220b48119c912562849a0528f35bfb592a9f72737 diff --git a/packages/pandas-gbq/.kokoro/docs/common.cfg b/packages/pandas-gbq/.kokoro/docs/common.cfg index 9102163e5ab6..09818f6aa05b 100644 --- a/packages/pandas-gbq/.kokoro/docs/common.cfg +++ b/packages/pandas-gbq/.kokoro/docs/common.cfg @@ -30,7 +30,9 @@ env_vars: { env_vars: { key: "V2_STAGING_BUCKET" - value: "docs-staging-v2" + # Push non-cloud library docs to `docs-staging-v2-staging` instead of the + # Cloud RAD bucket `docs-staging-v2` + value: "docs-staging-v2-staging" } # It will upload the docker image after successful builds. From 29858b485097141ca805964a28cb08a73d2b518c Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 2 Nov 2021 09:51:59 -0500 Subject: [PATCH 240/519] feat: `to_gbq` uses Parquet by default, use `api_method="load_csv"` for old behavior (#413) * avoid parquet for older pandas docs: deprecate `chunksize` when used with load jobs * keep `chunksize` for future use in streaming APIs deps: explicitly require `pyarrow >= 3.0` * mention pyarrow as a dependency * add pyarrow to conda deps deps: explicitly require `numpy >= 1.16.6` * update minimum numpy Co-authored-by: Owl Bot --- packages/pandas-gbq/CONTRIBUTING.rst | 2 +- .../ci/requirements-3.7-0.23.2.conda | 3 +- .../ci/requirements-3.9-NIGHTLY.conda | 1 + packages/pandas-gbq/docs/install.rst | 3 +- packages/pandas-gbq/noxfile.py | 2 +- packages/pandas-gbq/owlbot.py | 2 +- packages/pandas-gbq/pandas_gbq/exceptions.py | 15 +- packages/pandas-gbq/pandas_gbq/features.py | 10 ++ packages/pandas-gbq/pandas_gbq/gbq.py | 49 ++++-- packages/pandas-gbq/pandas_gbq/load.py | 141 ++++++++++++++---- packages/pandas-gbq/setup.py | 2 + .../pandas-gbq/testing/constraints-3.7.txt | 5 +- packages/pandas-gbq/tests/system/test_gbq.py | 74 +-------- .../pandas-gbq/tests/system/test_to_gbq.py | 77 +++++++--- packages/pandas-gbq/tests/unit/test_gbq.py | 23 +++ packages/pandas-gbq/tests/unit/test_load.py | 28 +++- 16 files changed, 286 insertions(+), 151 deletions(-) diff --git a/packages/pandas-gbq/CONTRIBUTING.rst b/packages/pandas-gbq/CONTRIBUTING.rst index 3c7bb6217a44..bc37b498e1cd 100644 --- a/packages/pandas-gbq/CONTRIBUTING.rst +++ b/packages/pandas-gbq/CONTRIBUTING.rst @@ -148,7 +148,7 @@ Running System Tests .. note:: - System tests are only configured to run under Python 3.8 and 3.9. + System tests are only configured to run under Python 3.7, 3.8 and 3.9. For expediency, we do not run them in older versions of Python 3. This alone will not run the tests. You'll need to change some local diff --git a/packages/pandas-gbq/ci/requirements-3.7-0.23.2.conda b/packages/pandas-gbq/ci/requirements-3.7-0.23.2.conda index af4768ab3405..1da6d2262f62 100644 --- a/packages/pandas-gbq/ci/requirements-3.7-0.23.2.conda +++ b/packages/pandas-gbq/ci/requirements-3.7-0.23.2.conda @@ -2,8 +2,9 @@ codecov coverage fastavro flake8 -numpy==1.14.5 +numpy==1.16.6 google-cloud-bigquery==1.11.1 +pyarrow==3.0.0 pydata-google-auth pytest pytest-cov diff --git a/packages/pandas-gbq/ci/requirements-3.9-NIGHTLY.conda b/packages/pandas-gbq/ci/requirements-3.9-NIGHTLY.conda index 9dfe3f6b071f..ccaa87e534ea 100644 --- a/packages/pandas-gbq/ci/requirements-3.9-NIGHTLY.conda +++ b/packages/pandas-gbq/ci/requirements-3.9-NIGHTLY.conda @@ -1,6 +1,7 @@ pydata-google-auth google-cloud-bigquery google-cloud-bigquery-storage +pyarrow pytest pytest-cov codecov diff --git a/packages/pandas-gbq/docs/install.rst b/packages/pandas-gbq/docs/install.rst index 6810a44a51e9..9887c79962ed 100644 --- a/packages/pandas-gbq/docs/install.rst +++ b/packages/pandas-gbq/docs/install.rst @@ -29,7 +29,7 @@ Install from Source .. code-block:: shell - $ pip install git+https://github.com/pydata/pandas-gbq.git + $ pip install git+https://github.com/googleapis/python-bigquery-pandas.git Dependencies @@ -38,6 +38,7 @@ Dependencies This module requires following additional dependencies: - `pydata-google-auth `__: Helpers for authentication to Google's API +- `pyarrow `__: Format for getting data to/from Google BigQuery - `google-auth `__: authentication and authorization for Google's API - `google-auth-oauthlib `__: integration with `oauthlib `__ for end-user authentication - `google-cloud-bigquery `__: Google Cloud client library for BigQuery diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 2542369dabec..ed88b094f7ed 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -28,7 +28,7 @@ BLACK_PATHS = ["docs", "pandas_gbq", "tests", "noxfile.py", "setup.py"] DEFAULT_PYTHON_VERSION = "3.8" -SYSTEM_TEST_PYTHON_VERSIONS = ["3.8", "3.9"] +SYSTEM_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9"] UNIT_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9"] CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() diff --git a/packages/pandas-gbq/owlbot.py b/packages/pandas-gbq/owlbot.py index d382cf6658e3..76a17e4012a9 100644 --- a/packages/pandas-gbq/owlbot.py +++ b/packages/pandas-gbq/owlbot.py @@ -31,7 +31,7 @@ extras = ["tqdm"] templated_files = common.py_library( unit_test_python_versions=["3.7", "3.8", "3.9"], - system_test_python_versions=["3.8", "3.9"], + system_test_python_versions=["3.7", "3.8", "3.9"], cov_level=86, unit_test_extras=extras, system_test_extras=extras, diff --git a/packages/pandas-gbq/pandas_gbq/exceptions.py b/packages/pandas-gbq/pandas_gbq/exceptions.py index aec0ea1a2577..1b4f6925097a 100644 --- a/packages/pandas-gbq/pandas_gbq/exceptions.py +++ b/packages/pandas-gbq/pandas_gbq/exceptions.py @@ -3,12 +3,23 @@ # license that can be found in the LICENSE file. +class GenericGBQException(ValueError): + """ + Raised when an unrecognized Google API Error occurs. + """ + + class AccessDenied(ValueError): """ Raised when invalid credentials are provided, or tokens have expired. """ - pass + +class ConversionError(GenericGBQException): + """ + Raised when there is a problem converting the DataFrame to a format + required to upload it to BigQuery. + """ class InvalidPrivateKeyFormat(ValueError): @@ -16,8 +27,6 @@ class InvalidPrivateKeyFormat(ValueError): Raised when provided private key has invalid format. """ - pass - class PerformanceWarning(RuntimeWarning): """ diff --git a/packages/pandas-gbq/pandas_gbq/features.py b/packages/pandas-gbq/pandas_gbq/features.py index ef1969fd6407..fc8ef5684548 100644 --- a/packages/pandas-gbq/pandas_gbq/features.py +++ b/packages/pandas-gbq/pandas_gbq/features.py @@ -10,6 +10,7 @@ BIGQUERY_BQSTORAGE_VERSION = "1.24.0" BIGQUERY_FROM_DATAFRAME_CSV_VERSION = "2.6.0" PANDAS_VERBOSITY_DEPRECATION_VERSION = "0.23.0" +PANDAS_PARQUET_LOSSLESS_TIMESTAMP_VERSION = "1.1.0" class Features: @@ -89,5 +90,14 @@ def pandas_has_deprecated_verbose(self): ) return self.pandas_installed_version >= pandas_verbosity_deprecation + @property + def pandas_has_parquet_with_lossless_timestamp(self): + import pkg_resources + + desired_version = pkg_resources.parse_version( + PANDAS_PARQUET_LOSSLESS_TIMESTAMP_VERSION + ) + return self.pandas_installed_version >= desired_version + FEATURES = Features() diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 856c128529c6..5c6ae4570af8 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -18,8 +18,11 @@ bigquery = None google_exceptions = None -from pandas_gbq.exceptions import AccessDenied -from pandas_gbq.exceptions import PerformanceWarning +from pandas_gbq.exceptions import ( + AccessDenied, + GenericGBQException, + PerformanceWarning, +) from pandas_gbq import features from pandas_gbq.features import FEATURES import pandas_gbq.schema @@ -69,14 +72,6 @@ class DatasetCreationError(ValueError): pass -class GenericGBQException(ValueError): - """ - Raised when an unrecognized Google API Error occurs. - """ - - pass - - class InvalidColumnOrder(ValueError): """ Raised when the provided column order for output @@ -520,7 +515,7 @@ def _download_results( df = rows_iter.to_dataframe( dtypes=conversion_dtypes, progress_bar_type=progress_bar_type, - **to_dataframe_kwargs + **to_dataframe_kwargs, ) except self.http_error as ex: self.process_http_error(ex) @@ -541,6 +536,7 @@ def load_data( chunksize=None, schema=None, progress_bar=True, + api_method: str = "load_parquet", ): from pandas_gbq import load @@ -554,6 +550,7 @@ def load_data( chunksize=chunksize, schema=schema, location=self.location, + api_method=api_method, ) if progress_bar and tqdm: chunks = tqdm.tqdm(chunks) @@ -876,6 +873,7 @@ def to_gbq( location=None, progress_bar=True, credentials=None, + api_method: str = "default", verbose=None, private_key=None, ): @@ -964,6 +962,12 @@ def to_gbq( :class:`google.oauth2.service_account.Credentials` directly. .. versionadded:: 0.8.0 + api_method : str, optional + API method used to upload DataFrame to BigQuery. One of "load_parquet", + "load_csv". Default "load_parquet" if pandas is version 1.1.0+, + otherwise "load_csv". + + .. versionadded:: 0.16.0 verbose : bool, deprecated Deprecated in Pandas-GBQ 0.4.0. Use the `logging module to adjust verbosity instead @@ -988,6 +992,28 @@ def to_gbq( stacklevel=1, ) + if api_method == "default": + # Avoid using parquet if pandas doesn't support lossless conversions to + # parquet timestamp. See: https://stackoverflow.com/a/69758676/101923 + if FEATURES.pandas_has_parquet_with_lossless_timestamp: + api_method = "load_parquet" + else: + api_method = "load_csv" + + if chunksize is not None: + if api_method == "load_parquet": + warnings.warn( + "chunksize is ignored when using api_method='load_parquet'", + DeprecationWarning, + stacklevel=2, + ) + elif api_method == "load_csv": + warnings.warn( + "chunksize will be ignored when using api_method='load_csv' in a future version of pandas-gbq", + PendingDeprecationWarning, + stacklevel=2, + ) + if if_exists not in ("fail", "replace", "append"): raise ValueError("'{0}' is not valid for if_exists".format(if_exists)) @@ -1071,6 +1097,7 @@ def to_gbq( chunksize=chunksize, schema=table_schema, progress_bar=progress_bar, + api_method=api_method, ) diff --git a/packages/pandas-gbq/pandas_gbq/load.py b/packages/pandas-gbq/pandas_gbq/load.py index faa674c21d5c..69210e41bff6 100644 --- a/packages/pandas-gbq/pandas_gbq/load.py +++ b/packages/pandas-gbq/pandas_gbq/load.py @@ -5,9 +5,13 @@ """Helper methods for loading data into BigQuery""" import io +from typing import Any, Callable, Dict, List, Optional +import pandas +import pyarrow.lib from google.cloud import bigquery +from pandas_gbq import exceptions from pandas_gbq.features import FEATURES import pandas_gbq.schema @@ -52,45 +56,126 @@ def split_dataframe(dataframe, chunksize=None): yield remaining_rows, chunk -def load_chunks( - client, - dataframe, - destination_table_ref, - chunksize=None, - schema=None, - location=None, +def load_parquet( + client: bigquery.Client, + dataframe: pandas.DataFrame, + destination_table_ref: bigquery.TableReference, + location: Optional[str], + schema: Optional[Dict[str, Any]], ): job_config = bigquery.LoadJobConfig() job_config.write_disposition = "WRITE_APPEND" - job_config.source_format = "CSV" - job_config.allow_quoted_newlines = True + job_config.source_format = "PARQUET" - # Explicit schema? Use that! if schema is not None: schema = pandas_gbq.schema.remove_policy_tags(schema) job_config.schema = pandas_gbq.schema.to_google_cloud_bigquery(schema) - # If not, let BigQuery determine schema unless we are encoding the CSV files ourselves. - elif not FEATURES.bigquery_has_from_dataframe_with_csv: - schema = pandas_gbq.schema.generate_bq_schema(dataframe) - schema = pandas_gbq.schema.remove_policy_tags(schema) - job_config.schema = pandas_gbq.schema.to_google_cloud_bigquery(schema) + try: + client.load_table_from_dataframe( + dataframe, destination_table_ref, job_config=job_config, location=location, + ).result() + except pyarrow.lib.ArrowInvalid as exc: + raise exceptions.ConversionError( + "Could not convert DataFrame to Parquet." + ) from exc + + +def load_csv( + dataframe: pandas.DataFrame, + chunksize: Optional[int], + bq_schema: Optional[List[bigquery.SchemaField]], + load_chunk: Callable, +): + job_config = bigquery.LoadJobConfig() + job_config.write_disposition = "WRITE_APPEND" + job_config.source_format = "CSV" + job_config.allow_quoted_newlines = True + + if bq_schema is not None: + job_config.schema = bq_schema + + # TODO: Remove chunking feature for load jobs. Deprecated in 0.16.0. chunks = split_dataframe(dataframe, chunksize=chunksize) for remaining_rows, chunk in chunks: yield remaining_rows + load_chunk(chunk, job_config) - if FEATURES.bigquery_has_from_dataframe_with_csv: - client.load_table_from_dataframe( - chunk, destination_table_ref, job_config=job_config, location=location, + +def load_csv_from_dataframe( + client: bigquery.Client, + dataframe: pandas.DataFrame, + destination_table_ref: bigquery.TableReference, + location: Optional[str], + chunksize: Optional[int], + schema: Optional[Dict[str, Any]], +): + bq_schema = None + + if schema is not None: + schema = pandas_gbq.schema.remove_policy_tags(schema) + bq_schema = pandas_gbq.schema.to_google_cloud_bigquery(schema) + + def load_chunk(chunk, job_config): + client.load_table_from_dataframe( + chunk, destination_table_ref, job_config=job_config, location=location, + ).result() + + return load_csv(dataframe, chunksize, bq_schema, load_chunk) + + +def load_csv_from_file( + client: bigquery.Client, + dataframe: pandas.DataFrame, + destination_table_ref: bigquery.TableReference, + location: Optional[str], + chunksize: Optional[int], + schema: Optional[Dict[str, Any]], +): + if schema is None: + schema = pandas_gbq.schema.generate_bq_schema(dataframe) + + schema = pandas_gbq.schema.remove_policy_tags(schema) + bq_schema = pandas_gbq.schema.to_google_cloud_bigquery(schema) + + def load_chunk(chunk, job_config): + try: + chunk_buffer = encode_chunk(chunk) + client.load_table_from_file( + chunk_buffer, + destination_table_ref, + job_config=job_config, + location=location, ).result() + finally: + chunk_buffer.close() + + return load_csv(dataframe, chunksize, bq_schema, load_chunk,) + + +def load_chunks( + client, + dataframe, + destination_table_ref, + chunksize=None, + schema=None, + location=None, + api_method="load_parquet", +): + if api_method == "load_parquet": + load_parquet(client, dataframe, destination_table_ref, location, schema) + # TODO: yield progress depending on result() with timeout + return [0] + elif api_method == "load_csv": + if FEATURES.bigquery_has_from_dataframe_with_csv: + return load_csv_from_dataframe( + client, dataframe, destination_table_ref, location, chunksize, schema + ) else: - try: - chunk_buffer = encode_chunk(chunk) - client.load_table_from_file( - chunk_buffer, - destination_table_ref, - job_config=job_config, - location=location, - ).result() - finally: - chunk_buffer.close() + return load_csv_from_file( + client, dataframe, destination_table_ref, location, chunksize, schema + ) + else: + raise ValueError( + f"Got unexpected api_method: {api_method!r}, expected one of 'load_parquet', 'load_csv'." + ) diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index a7e23eece824..b1169cef7ee2 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -23,7 +23,9 @@ release_status = "Development Status :: 4 - Beta" dependencies = [ "setuptools", + "numpy>=1.16.6", "pandas>=0.23.2", + "pyarrow >=3.0.0, <7.0dev", "pydata-google-auth", "google-auth", "google-auth-oauthlib", diff --git a/packages/pandas-gbq/testing/constraints-3.7.txt b/packages/pandas-gbq/testing/constraints-3.7.txt index 251c81b4492a..7c67d2750fe2 100644 --- a/packages/pandas-gbq/testing/constraints-3.7.txt +++ b/packages/pandas-gbq/testing/constraints-3.7.txt @@ -5,11 +5,12 @@ # # e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", # Then this file should have foo==1.14.0 -numpy==1.14.5 -pandas==0.23.2 google-auth==1.4.1 google-auth-oauthlib==0.0.1 google-cloud-bigquery==1.11.1 google-cloud-bigquery-storage==1.1.0 +numpy==1.16.6 +pandas==0.23.2 +pyarrow==3.0.0 pydata-google-auth==0.1.2 tqdm==4.23.0 diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index 00bbd3d6b16c..a8d6bd0ddc1b 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -1185,77 +1185,6 @@ def test_google_upload_errors_should_raise_exception(self, project_id): credentials=self.credentials, ) - def test_upload_chinese_unicode_data(self, project_id): - test_id = "2" - test_size = 6 - df = DataFrame(np.random.randn(6, 4), index=range(6), columns=list("ABCD")) - df["s"] = u"信用卡" - - gbq.to_gbq( - df, - self.destination_table + test_id, - project_id, - credentials=self.credentials, - chunksize=10000, - ) - - result_df = gbq.read_gbq( - "SELECT * FROM {0}".format(self.destination_table + test_id), - project_id=project_id, - credentials=self.credentials, - dialect="legacy", - ) - - assert len(result_df) == test_size - - if sys.version_info.major < 3: - pytest.skip(msg="Unicode comparison in Py2 not working") - - result = result_df["s"].sort_values() - expected = df["s"].sort_values() - - tm.assert_numpy_array_equal(expected.values, result.values) - - def test_upload_other_unicode_data(self, project_id): - test_id = "3" - test_size = 3 - df = DataFrame( - { - "s": ["Skywalker™", "lego", "hülle"], - "i": [200, 300, 400], - "d": [ - "2017-12-13 17:40:39", - "2017-12-13 17:40:39", - "2017-12-13 17:40:39", - ], - } - ) - - gbq.to_gbq( - df, - self.destination_table + test_id, - project_id=project_id, - credentials=self.credentials, - chunksize=10000, - ) - - result_df = gbq.read_gbq( - "SELECT * FROM {0}".format(self.destination_table + test_id), - project_id=project_id, - credentials=self.credentials, - dialect="legacy", - ) - - assert len(result_df) == test_size - - if sys.version_info.major < 3: - pytest.skip(msg="Unicode comparison in Py2 not working") - - result = result_df["s"].sort_values() - expected = df["s"].sort_values() - - tm.assert_numpy_array_equal(expected.values, result.values) - def test_upload_mixed_float_and_int(self, project_id): """Test that we can upload a dataframe containing an int64 and float64 column. See: https://github.com/pydata/pandas-gbq/issues/116 @@ -1454,6 +1383,9 @@ def test_upload_data_with_different_df_and_user_schema(self, project_id): project_id, credentials=self.credentials, table_schema=test_schema, + # Loading string pandas series to FLOAT column not supported with + # Parquet. + api_method="load_csv", ) dataset, table = destination_table.split(".") assert verify_schema( diff --git a/packages/pandas-gbq/tests/system/test_to_gbq.py b/packages/pandas-gbq/tests/system/test_to_gbq.py index b0d2d031db8f..d16997fda287 100644 --- a/packages/pandas-gbq/tests/system/test_to_gbq.py +++ b/packages/pandas-gbq/tests/system/test_to_gbq.py @@ -3,9 +3,10 @@ # license that can be found in the LICENSE file. import functools +import random + import pandas import pandas.testing - import pytest @@ -21,31 +22,61 @@ def method_under_test(credentials, project_id): ) -def test_float_round_trip(method_under_test, random_dataset_id, bigquery_client): - """Ensure that 64-bit floating point numbers are unchanged. - - See: https://github.com/pydata/pandas-gbq/issues/326 - """ - - table_id = "{}.float_round_trip".format(random_dataset_id) - input_floats = pandas.Series( - [ - 0.14285714285714285, - 0.4406779661016949, - 1.05148, - 1.05153, - 1.8571428571428572, - 2.718281828459045, - 3.141592653589793, - 2.0988936657440586e43, - ], - name="float_col", +@pytest.mark.parametrize( + ["input_series"], + [ + # Ensure that 64-bit floating point numbers are unchanged. + # See: https://github.com/pydata/pandas-gbq/issues/326 + ( + pandas.Series( + [ + 0.14285714285714285, + 0.4406779661016949, + 1.05148, + 1.05153, + 1.8571428571428572, + 2.718281828459045, + 3.141592653589793, + 2.0988936657440586e43, + ], + name="test_col", + ), + ), + ( + pandas.Series( + [ + "abc", + "defg", + # Ensure that empty strings are written as empty string, + # not NULL. See: + # https://github.com/googleapis/python-bigquery-pandas/issues/366 + "", + None, + # Ensure that unicode characters are encoded. See: + # https://github.com/googleapis/python-bigquery-pandas/issues/106 + "信用卡", + "Skywalker™", + "hülle", + ], + name="test_col", + ), + ), + ], +) +def test_series_round_trip( + method_under_test, random_dataset_id, bigquery_client, input_series +): + table_id = f"{random_dataset_id}.round_trip_{random.randrange(1_000_000)}" + input_series = input_series.sort_values().reset_index(drop=True) + df = pandas.DataFrame( + # Some errors only occur in multi-column dataframes. See: + # https://github.com/googleapis/python-bigquery-pandas/issues/366 + {"test_col": input_series, "test_col2": input_series} ) - df = pandas.DataFrame({"float_col": input_floats}) method_under_test(df, table_id) round_trip = bigquery_client.list_rows(table_id).to_dataframe() - round_trip_floats = round_trip["float_col"].sort_values() + round_trip_series = round_trip["test_col"].sort_values().reset_index(drop=True) pandas.testing.assert_series_equal( - round_trip_floats, input_floats, check_exact=True, + round_trip_series, input_series, check_exact=True, ) diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index 3b6034128604..0a5ecad24532 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -124,6 +124,29 @@ def test_to_gbq_with_no_project_id_given_should_fail(monkeypatch): gbq.to_gbq(DataFrame([[1]]), "dataset.tablename") +@pytest.mark.parametrize( + ["api_method", "warning_message", "warning_type"], + [ + ("load_parquet", "chunksize is ignored", DeprecationWarning), + ("load_csv", "chunksize will be ignored", PendingDeprecationWarning), + ], +) +def test_to_gbq_with_chunksize_warns_deprecation( + api_method, warning_message, warning_type +): + with pytest.warns(warning_type, match=warning_message): + try: + gbq.to_gbq( + DataFrame([[1]]), + "dataset.tablename", + project_id="my-project", + api_method=api_method, + chunksize=100, + ) + except gbq.TableCreationError: + pass + + @pytest.mark.parametrize(["verbose"], [(True,), (False,)]) def test_to_gbq_with_verbose_new_pandas_warns_deprecation(monkeypatch, verbose): monkeypatch.setattr( diff --git a/packages/pandas-gbq/tests/unit/test_load.py b/packages/pandas-gbq/tests/unit/test_load.py index 2ddb4f50c84d..a32d2d9e62ea 100644 --- a/packages/pandas-gbq/tests/unit/test_load.py +++ b/packages/pandas-gbq/tests/unit/test_load.py @@ -16,10 +16,10 @@ from pandas_gbq import load -def load_method(bqclient): - if FEATURES.bigquery_has_from_dataframe_with_csv: - return bqclient.load_table_from_dataframe - return bqclient.load_table_from_file +def load_method(bqclient, api_method): + if not FEATURES.bigquery_has_from_dataframe_with_csv and api_method == "load_csv": + return bqclient.load_table_from_file + return bqclient.load_table_from_dataframe def test_encode_chunk_with_unicode(): @@ -91,9 +91,12 @@ def test_encode_chunks_with_chunksize_none(): assert len(chunk.index) == 6 -@pytest.mark.parametrize(["bigquery_has_from_dataframe_with_csv"], [(True,), (False,)]) +@pytest.mark.parametrize( + ["bigquery_has_from_dataframe_with_csv", "api_method"], + [(True, "load_parquet"), (True, "load_csv"), (False, "load_csv")], +) def test_load_chunks_omits_policy_tags( - monkeypatch, mock_bigquery_client, bigquery_has_from_dataframe_with_csv + monkeypatch, mock_bigquery_client, bigquery_has_from_dataframe_with_csv, api_method ): """Ensure that policyTags are omitted. @@ -117,11 +120,20 @@ def test_load_chunks_omits_policy_tags( ] } - _ = list(load.load_chunks(mock_bigquery_client, df, destination, schema=schema)) + _ = list( + load.load_chunks( + mock_bigquery_client, df, destination, schema=schema, api_method=api_method + ) + ) - mock_load = load_method(mock_bigquery_client) + mock_load = load_method(mock_bigquery_client, api_method=api_method) assert mock_load.called _, kwargs = mock_load.call_args assert "job_config" in kwargs sent_field = kwargs["job_config"].schema[0].to_api_repr() assert "policyTags" not in sent_field + + +def test_load_chunks_with_invalid_api_method(): + with pytest.raises(ValueError, match="Got unexpected api_method:"): + load.load_chunks(None, None, None, api_method="not_a_thing") From cf61c418e31ad6e297089b4960002d2be9292430 Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Tue, 2 Nov 2021 16:26:14 -0600 Subject: [PATCH 241/519] docs: clarify `table_schema` (#383) 1. There must be an input dataframe column for every desired output table column. 2. The types given in table_schema must be BigQuery types (strings). - [x] closes #382 - [x] tests added / passed - [x] passes `nox -s blacken lint` --- packages/pandas-gbq/pandas_gbq/gbq.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 5c6ae4570af8..87c2327c9ccc 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -927,11 +927,12 @@ def to_gbq( table_schema : list of dicts, optional List of BigQuery table fields to which according DataFrame columns conform to, e.g. ``[{'name': 'col1', 'type': - 'STRING'},...]``. + 'STRING'},...]``. The ``type`` values must be BigQuery type names. - If ``table_schema`` is provided, it may contain all or a subset of DataFrame columns. If a subset is provided, the rest will be - inferred from the DataFrame dtypes. + inferred from the DataFrame dtypes. If ``table_schema`` contains + columns not in the DataFrame, they'll be ignored. - If ``table_schema`` is **not** provided, it will be generated according to dtypes of DataFrame columns. See `Inferring the Table Schema From 47ca7d11e94c8efc7a043647dc3d0e5bc10298b4 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Mon, 8 Nov 2021 18:01:20 +0100 Subject: [PATCH 242/519] chore(deps): update all dependencies (#403) * chore(deps): update all dependencies * update bigquery * update pandas * update pyarrow Co-authored-by: Anthonios Partheniou Co-authored-by: Tim Swast --- packages/pandas-gbq/samples/snippets/requirements.txt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index cf8cecae3a64..9f5ef29d1656 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,5 +1,4 @@ -google-cloud-bigquery-storage==2.8.0 -google-cloud-bigquery==2.27.0 -pandas==1.3.1 -pandas-gbq==0.15.0 -pyarrow==5.0.0 +google-cloud-bigquery-storage==2.10.0 +google-cloud-bigquery==2.30.1 +pandas==1.3.4 +pyarrow==6.0.0 From 7b61b8b2d1062349a578352ec009faf8bf2ef07f Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 8 Nov 2021 12:14:16 -0600 Subject: [PATCH 243/519] feat: allow Python 3.10 (#417) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: allow Python 3.10 * add missing test configs * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot --- packages/pandas-gbq/setup.py | 3 ++- packages/pandas-gbq/testing/constraints-3.10.txt | 0 packages/pandas-gbq/testing/constraints-3.11.txt | 0 3 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 packages/pandas-gbq/testing/constraints-3.10.txt create mode 100644 packages/pandas-gbq/testing/constraints-3.11.txt diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index b1169cef7ee2..b66c04997c1c 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -77,6 +77,7 @@ "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Operating System :: OS Independent", "Topic :: Internet", "Topic :: Scientific/Engineering", @@ -85,7 +86,7 @@ packages=packages, install_requires=dependencies, extras_require=extras, - python_requires=">=3.7, <3.10", + python_requires=">=3.7, <3.11", include_package_data=True, zip_safe=False, ) diff --git a/packages/pandas-gbq/testing/constraints-3.10.txt b/packages/pandas-gbq/testing/constraints-3.10.txt new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/packages/pandas-gbq/testing/constraints-3.11.txt b/packages/pandas-gbq/testing/constraints-3.11.txt new file mode 100644 index 000000000000..e69de29bb2d1 From 674fa88d7fa216653fe25d0c5193db01b03a3760 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 8 Nov 2021 12:40:12 -0600 Subject: [PATCH 244/519] chore: release 0.16.0 (#416) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- packages/pandas-gbq/CHANGELOG.md | 18 ++++++++++++++++++ packages/pandas-gbq/pandas_gbq/version.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index dd258c577e6a..318b52410ba8 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## [0.16.0](https://www.github.com/googleapis/python-bigquery-pandas/compare/v0.16.0...v0.16.0) (2021-11-08) + + +### Features + +* `to_gbq` uses Parquet by default, use `api_method="load_csv"` for old behavior ([#413](https://www.github.com/googleapis/python-bigquery-pandas/issues/413)) ([9a65383](https://www.github.com/googleapis/python-bigquery-pandas/commit/9a65383916697ff02358aba58df555c85b16350c)) +* allow Python 3.10 ([#417](https://www.github.com/googleapis/python-bigquery-pandas/issues/417)) ([faba940](https://www.github.com/googleapis/python-bigquery-pandas/commit/faba940bc19d5c260b9dce3f973a9b729a179d20)) + + +### Miscellaneous Chores + +* release 0.16.0 ([#415](https://www.github.com/googleapis/python-bigquery-pandas/issues/415)) ([ea0f4e9](https://www.github.com/googleapis/python-bigquery-pandas/commit/ea0f4e97f3518895e824b1b7328d578081588d84)) + + +### Documentation + +* clarify `table_schema` ([#383](https://www.github.com/googleapis/python-bigquery-pandas/issues/383)) ([326e674](https://www.github.com/googleapis/python-bigquery-pandas/commit/326e674a24fc7e057e213596df92f0c4a8225f9e)) + ## 0.15.0 / 2021-03-30 ### Features diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py index 4094268e403f..057137c2c114 100644 --- a/packages/pandas-gbq/pandas_gbq/version.py +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -2,4 +2,4 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -__version__ = "0.15.0" +__version__ = "0.16.0" From e79f20da33fcfb13341a6b9dbd16ae1ed8dd7c8e Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 11 Nov 2021 13:26:24 -0500 Subject: [PATCH 245/519] chore(python): add .github/CODEOWNERS as a templated file (#422) Source-Link: https://github.com/googleapis/synthtool/commit/c5026b3217973a8db55db8ee85feee0e9a65e295 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:0e18b9475fbeb12d9ad4302283171edebb6baf2dfca1bd215ee3b34ed79d95d7 Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 2 +- packages/pandas-gbq/.github/CODEOWNERS | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index cb89b2e326b7..7519fa3a2289 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:ec49167c606648a063d1222220b48119c912562849a0528f35bfb592a9f72737 + digest: sha256:0e18b9475fbeb12d9ad4302283171edebb6baf2dfca1bd215ee3b34ed79d95d7 diff --git a/packages/pandas-gbq/.github/CODEOWNERS b/packages/pandas-gbq/.github/CODEOWNERS index 3c9ab94c6c16..f8714a3e787d 100644 --- a/packages/pandas-gbq/.github/CODEOWNERS +++ b/packages/pandas-gbq/.github/CODEOWNERS @@ -3,9 +3,10 @@ # # For syntax help see: # https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners#codeowners-syntax +# Note: This file is autogenerated. To make changes to the codeowner team, please update .repo-metadata.json. -# The @googleapis/yoshi-python is the default owner for changes in this repo -* @googleapis/api-bigquery @googleapis/yoshi-python +# @googleapis/yoshi-python @googleapis/api-bigquery are the default owners for changes in this repo +* @googleapis/yoshi-python @googleapis/api-bigquery -# The python-samples-reviewers team is the default owner for samples changes -/samples/ @googleapis/api-bigquery @googleapis/python-samples-owners +# @googleapis/python-samples-owners @googleapis/api-bigquery are the default owners for samples changes +/samples/ @googleapis/python-samples-owners @googleapis/api-bigquery From 60cbb223df6053cf6c654bb1e9c803ffe1e75b0c Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 16 Nov 2021 16:55:58 -0600 Subject: [PATCH 246/519] test: upload DATE column with various dtypes (#420) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test: upload DATE column with various dtypes * add dbdate tests * test with db-dtypes only with newer pandas * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * sort by row number Co-authored-by: Owl Bot --- packages/pandas-gbq/CONTRIBUTING.rst | 10 +- packages/pandas-gbq/noxfile.py | 10 +- packages/pandas-gbq/owlbot.py | 8 +- packages/pandas-gbq/setup.py | 5 +- .../pandas-gbq/tests/system/test_to_gbq.py | 106 ++++++++++++++++-- 5 files changed, 120 insertions(+), 19 deletions(-) diff --git a/packages/pandas-gbq/CONTRIBUTING.rst b/packages/pandas-gbq/CONTRIBUTING.rst index bc37b498e1cd..90bd84f2e34b 100644 --- a/packages/pandas-gbq/CONTRIBUTING.rst +++ b/packages/pandas-gbq/CONTRIBUTING.rst @@ -22,7 +22,7 @@ In order to add a feature: documentation. - The feature must work fully on the following CPython versions: - 3.7, 3.8 and 3.9 on both UNIX and Windows. + 3.7, 3.8, 3.9 and 3.10 on both UNIX and Windows. - The feature must not add unnecessary dependencies (where "unnecessary" is of course subjective, but new dependencies should @@ -72,7 +72,7 @@ We use `nox `__ to instrument our tests. - To run a single unit test:: - $ nox -s unit-3.9 -- -k + $ nox -s unit-3.10 -- -k .. note:: @@ -143,12 +143,12 @@ Running System Tests $ nox -s system # Run a single system test - $ nox -s system-3.9 -- -k + $ nox -s system-3.10 -- -k .. note:: - System tests are only configured to run under Python 3.7, 3.8 and 3.9. + System tests are only configured to run under Python 3.7, 3.8, 3.9 and 3.10. For expediency, we do not run them in older versions of Python 3. This alone will not run the tests. You'll need to change some local @@ -224,10 +224,12 @@ We support: - `Python 3.7`_ - `Python 3.8`_ - `Python 3.9`_ +- `Python 3.10`_ .. _Python 3.7: https://docs.python.org/3.7/ .. _Python 3.8: https://docs.python.org/3.8/ .. _Python 3.9: https://docs.python.org/3.9/ +.. _Python 3.10: https://docs.python.org/3.10/ Supported versions can be found in our ``noxfile.py`` `config`_. diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index ed88b094f7ed..825daf18ac46 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -28,8 +28,8 @@ BLACK_PATHS = ["docs", "pandas_gbq", "tests", "noxfile.py", "setup.py"] DEFAULT_PYTHON_VERSION = "3.8" -SYSTEM_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9"] -UNIT_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9"] +SYSTEM_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] +UNIT_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() @@ -146,7 +146,11 @@ def system(session): # Install all test dependencies, then install this package into the # virtualenv's dist-packages. session.install("mock", "pytest", "google-cloud-testutils", "-c", constraints_path) - session.install("-e", ".[tqdm]", "-c", constraints_path) + if session.python == "3.9": + extras = "[tqdm,db-dtypes]" + else: + extras = "[tqdm]" + session.install("-e", f".{extras}", "-c", constraints_path) # Run py.test against the system tests. if system_test_exists: diff --git a/packages/pandas-gbq/owlbot.py b/packages/pandas-gbq/owlbot.py index 76a17e4012a9..71679dd4e7a6 100644 --- a/packages/pandas-gbq/owlbot.py +++ b/packages/pandas-gbq/owlbot.py @@ -29,12 +29,16 @@ # ---------------------------------------------------------------------------- extras = ["tqdm"] +extras_by_python = { + "3.9": ["tqdm", "db-dtypes"], +} templated_files = common.py_library( - unit_test_python_versions=["3.7", "3.8", "3.9"], - system_test_python_versions=["3.7", "3.8", "3.9"], + unit_test_python_versions=["3.7", "3.8", "3.9", "3.10"], + system_test_python_versions=["3.7", "3.8", "3.9", "3.10"], cov_level=86, unit_test_extras=extras, system_test_extras=extras, + system_test_extras_by_python=extras_by_python, intersphinx_dependencies={ "pandas": "https://pandas.pydata.org/pandas-docs/stable/", "pydata-google-auth": "https://pydata-google-auth.readthedocs.io/en/latest/", diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index b66c04997c1c..876bd4c09fa9 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -33,7 +33,10 @@ # https://github.com/pydata/pandas-gbq/issues/343 "google-cloud-bigquery[bqstorage,pandas]>=1.11.1,<3.0.0dev,!=2.4.*", ] -extras = {"tqdm": "tqdm>=4.23.0"} +extras = { + "tqdm": "tqdm>=4.23.0", + "db-dtypes": "db-dtypes >=0.3.0,<2.0.0", +} # Setup boilerplate below this line. diff --git a/packages/pandas-gbq/tests/system/test_to_gbq.py b/packages/pandas-gbq/tests/system/test_to_gbq.py index d16997fda287..4f315a77aacd 100644 --- a/packages/pandas-gbq/tests/system/test_to_gbq.py +++ b/packages/pandas-gbq/tests/system/test_to_gbq.py @@ -9,10 +9,20 @@ import pandas.testing import pytest +try: + import db_dtypes +except ImportError: + db_dtypes = None + pytest.importorskip("google.cloud.bigquery", minversion="1.24.0") +@pytest.fixture(params=["default", "load_parquet", "load_csv"]) +def api_method(request): + return request.param + + @pytest.fixture def method_under_test(credentials, project_id): import pandas_gbq @@ -23,7 +33,7 @@ def method_under_test(credentials, project_id): @pytest.mark.parametrize( - ["input_series"], + ["input_series", "skip_csv"], [ # Ensure that 64-bit floating point numbers are unchanged. # See: https://github.com/pydata/pandas-gbq/issues/326 @@ -41,17 +51,13 @@ def method_under_test(credentials, project_id): ], name="test_col", ), + False, ), ( pandas.Series( [ "abc", "defg", - # Ensure that empty strings are written as empty string, - # not NULL. See: - # https://github.com/googleapis/python-bigquery-pandas/issues/366 - "", - None, # Ensure that unicode characters are encoded. See: # https://github.com/googleapis/python-bigquery-pandas/issues/106 "信用卡", @@ -60,12 +66,35 @@ def method_under_test(credentials, project_id): ], name="test_col", ), + False, + ), + ( + pandas.Series( + [ + "abc", + "defg", + # Ensure that empty strings are written as empty string, + # not NULL. See: + # https://github.com/googleapis/python-bigquery-pandas/issues/366 + "", + None, + ], + name="empty_strings", + ), + True, ), ], ) def test_series_round_trip( - method_under_test, random_dataset_id, bigquery_client, input_series + method_under_test, + random_dataset_id, + bigquery_client, + input_series, + api_method, + skip_csv, ): + if api_method == "load_csv" and skip_csv: + pytest.skip("Loading with CSV not supported.") table_id = f"{random_dataset_id}.round_trip_{random.randrange(1_000_000)}" input_series = input_series.sort_values().reset_index(drop=True) df = pandas.DataFrame( @@ -73,10 +102,69 @@ def test_series_round_trip( # https://github.com/googleapis/python-bigquery-pandas/issues/366 {"test_col": input_series, "test_col2": input_series} ) - method_under_test(df, table_id) + method_under_test(df, table_id, api_method=api_method) round_trip = bigquery_client.list_rows(table_id).to_dataframe() round_trip_series = round_trip["test_col"].sort_values().reset_index(drop=True) pandas.testing.assert_series_equal( - round_trip_series, input_series, check_exact=True, + round_trip_series, input_series, check_exact=True, check_names=False, + ) + + +DATAFRAME_ROUND_TRIPS = [ + # Ensure that a DATE column can be written with datetime64[ns] dtype + # data. See: + # https://github.com/googleapis/python-bigquery-pandas/issues/362 + ( + pandas.DataFrame( + { + "date_col": pandas.Series( + ["2021-04-17", "1999-12-31", "2038-01-19"], dtype="datetime64[ns]", + ), + } + ), + [{"name": "date_col", "type": "DATE"}], + True, + ), +] +if db_dtypes is not None: + DATAFRAME_ROUND_TRIPS.append( + ( + pandas.DataFrame( + { + "date_col": pandas.Series( + ["2021-04-17", "1999-12-31", "2038-01-19"], dtype="dbdate", + ), + } + ), + [{"name": "date_col", "type": "DATE"}], + False, + ) + ) + + +@pytest.mark.parametrize( + ["input_df", "table_schema", "skip_csv"], DATAFRAME_ROUND_TRIPS +) +def test_dataframe_round_trip_with_table_schema( + method_under_test, + random_dataset_id, + bigquery_client, + input_df, + table_schema, + api_method, + skip_csv, +): + if api_method == "load_csv" and skip_csv: + pytest.skip("Loading with CSV not supported.") + table_id = f"{random_dataset_id}.round_trip_w_schema_{random.randrange(1_000_000)}" + input_df["row_num"] = input_df.index + input_df.sort_values("row_num", inplace=True) + method_under_test( + input_df, table_id, table_schema=table_schema, api_method=api_method + ) + round_trip = bigquery_client.list_rows(table_id).to_dataframe( + dtypes=dict(zip(input_df.columns, input_df.dtypes)) ) + round_trip.sort_values("row_num", inplace=True) + pandas.testing.assert_frame_equal(input_df, round_trip) From 8fde55cfcb74703c4393626ba038e9ede8447d98 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 22 Nov 2021 09:28:01 -0600 Subject: [PATCH 247/519] fix: `to_gbq` allows strings for DATE and floats for NUMERIC with `api_method="load_parquet"` (#423) deps: require pandas 0.24+ and db-dtypes for TIME/DATE extension dtypes (#423) --- packages/pandas-gbq/.circleci/config.yml | 2 +- packages/pandas-gbq/.coveragerc | 2 +- ....2.conda => requirements-3.7-0.24.2.conda} | 1 + .../ci/requirements-3.9-NIGHTLY.conda | 1 + packages/pandas-gbq/noxfile.py | 8 +- packages/pandas-gbq/owlbot.py | 6 +- packages/pandas-gbq/pandas_gbq/load.py | 52 +++++++ packages/pandas-gbq/setup.py | 4 +- .../pandas-gbq/testing/constraints-3.7.txt | 3 +- packages/pandas-gbq/tests/system/test_gbq.py | 21 +-- .../pandas-gbq/tests/system/test_to_gbq.py | 139 ++++++++++++------ packages/pandas-gbq/tests/unit/test_load.py | 120 ++++++++++++++- 12 files changed, 279 insertions(+), 80 deletions(-) rename packages/pandas-gbq/ci/{requirements-3.7-0.23.2.conda => requirements-3.7-0.24.2.conda} (89%) diff --git a/packages/pandas-gbq/.circleci/config.yml b/packages/pandas-gbq/.circleci/config.yml index ec4d7448a48f..4c378b3fb758 100644 --- a/packages/pandas-gbq/.circleci/config.yml +++ b/packages/pandas-gbq/.circleci/config.yml @@ -10,7 +10,7 @@ jobs: - image: continuumio/miniconda3 environment: PYTHON: "3.7" - PANDAS: "0.23.2" + PANDAS: "0.24.2" steps: - checkout - run: ci/config_auth.sh diff --git a/packages/pandas-gbq/.coveragerc b/packages/pandas-gbq/.coveragerc index 61285af5d8f2..ba50bf322ae0 100644 --- a/packages/pandas-gbq/.coveragerc +++ b/packages/pandas-gbq/.coveragerc @@ -22,7 +22,7 @@ omit = google/cloud/__init__.py [report] -fail_under = 86 +fail_under = 88 show_missing = True exclude_lines = # Re-enable the standard pragma diff --git a/packages/pandas-gbq/ci/requirements-3.7-0.23.2.conda b/packages/pandas-gbq/ci/requirements-3.7-0.24.2.conda similarity index 89% rename from packages/pandas-gbq/ci/requirements-3.7-0.23.2.conda rename to packages/pandas-gbq/ci/requirements-3.7-0.24.2.conda index 1da6d2262f62..82f4e7b9a454 100644 --- a/packages/pandas-gbq/ci/requirements-3.7-0.23.2.conda +++ b/packages/pandas-gbq/ci/requirements-3.7-0.24.2.conda @@ -1,5 +1,6 @@ codecov coverage +db-dtypes==0.3.0 fastavro flake8 numpy==1.16.6 diff --git a/packages/pandas-gbq/ci/requirements-3.9-NIGHTLY.conda b/packages/pandas-gbq/ci/requirements-3.9-NIGHTLY.conda index ccaa87e534ea..5a3e9fb72d21 100644 --- a/packages/pandas-gbq/ci/requirements-3.9-NIGHTLY.conda +++ b/packages/pandas-gbq/ci/requirements-3.9-NIGHTLY.conda @@ -1,3 +1,4 @@ +db-dtypes pydata-google-auth google-cloud-bigquery google-cloud-bigquery-storage diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 825daf18ac46..2feeccdc9f37 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -146,11 +146,7 @@ def system(session): # Install all test dependencies, then install this package into the # virtualenv's dist-packages. session.install("mock", "pytest", "google-cloud-testutils", "-c", constraints_path) - if session.python == "3.9": - extras = "[tqdm,db-dtypes]" - else: - extras = "[tqdm]" - session.install("-e", f".{extras}", "-c", constraints_path) + session.install("-e", ".[tqdm]", "-c", constraints_path) # Run py.test against the system tests. if system_test_exists: @@ -179,7 +175,7 @@ def cover(session): test runs (not system test runs), and then erases coverage data. """ session.install("coverage", "pytest-cov") - session.run("coverage", "report", "--show-missing", "--fail-under=86") + session.run("coverage", "report", "--show-missing", "--fail-under=88") session.run("coverage", "erase") diff --git a/packages/pandas-gbq/owlbot.py b/packages/pandas-gbq/owlbot.py index 71679dd4e7a6..c69d54deae56 100644 --- a/packages/pandas-gbq/owlbot.py +++ b/packages/pandas-gbq/owlbot.py @@ -29,16 +29,12 @@ # ---------------------------------------------------------------------------- extras = ["tqdm"] -extras_by_python = { - "3.9": ["tqdm", "db-dtypes"], -} templated_files = common.py_library( unit_test_python_versions=["3.7", "3.8", "3.9", "3.10"], system_test_python_versions=["3.7", "3.8", "3.9", "3.10"], - cov_level=86, + cov_level=88, unit_test_extras=extras, system_test_extras=extras, - system_test_extras_by_python=extras_by_python, intersphinx_dependencies={ "pandas": "https://pandas.pydata.org/pandas-docs/stable/", "pydata-google-auth": "https://pydata-google-auth.readthedocs.io/en/latest/", diff --git a/packages/pandas-gbq/pandas_gbq/load.py b/packages/pandas-gbq/pandas_gbq/load.py index 69210e41bff6..5422402e0e54 100644 --- a/packages/pandas-gbq/pandas_gbq/load.py +++ b/packages/pandas-gbq/pandas_gbq/load.py @@ -4,9 +4,11 @@ """Helper methods for loading data into BigQuery""" +import decimal import io from typing import Any, Callable, Dict, List, Optional +import db_dtypes import pandas import pyarrow.lib from google.cloud import bigquery @@ -56,6 +58,55 @@ def split_dataframe(dataframe, chunksize=None): yield remaining_rows, chunk +def cast_dataframe_for_parquet( + dataframe: pandas.DataFrame, schema: Optional[Dict[str, Any]], +) -> pandas.DataFrame: + """Cast columns to needed dtype when writing parquet files. + + See: https://github.com/googleapis/python-bigquery-pandas/issues/421 + """ + + columns = schema.get("fields", []) + + # Protect against an explicit None in the dictionary. + columns = columns if columns is not None else [] + + for column in columns: + # Schema can be a superset of the columns in the dataframe, so ignore + # columns that aren't present. + column_name = column.get("name") + if column_name not in dataframe.columns: + continue + + # Skip array columns for now. Potentially casting the elements of the + # array would be possible, but not worth the effort until there is + # demand for it. + if column.get("mode", "NULLABLE").upper() == "REPEATED": + continue + + column_type = column.get("type", "").upper() + if ( + column_type == "DATE" + # Use extension dtype first so that it uses the correct equality operator. + and db_dtypes.DateDtype() != dataframe[column_name].dtype + ): + # Construct converted column manually, because I can't use + # .astype() with DateDtype. With .astype(), I get the error: + # + # TypeError: Cannot interpret '' as a data type + cast_column = pandas.Series( + dataframe[column_name], dtype=db_dtypes.DateDtype() + ) + elif column_type in {"NUMERIC", "DECIMAL", "BIGNUMERIC", "BIGDECIMAL"}: + cast_column = dataframe[column_name].map(decimal.Decimal) + else: + cast_column = None + + if cast_column is not None: + dataframe = dataframe.assign(**{column_name: cast_column}) + return dataframe + + def load_parquet( client: bigquery.Client, dataframe: pandas.DataFrame, @@ -70,6 +121,7 @@ def load_parquet( if schema is not None: schema = pandas_gbq.schema.remove_policy_tags(schema) job_config.schema = pandas_gbq.schema.to_google_cloud_bigquery(schema) + dataframe = cast_dataframe_for_parquet(dataframe, schema) try: client.load_table_from_dataframe( diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index 876bd4c09fa9..28c81eee087f 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -23,8 +23,9 @@ release_status = "Development Status :: 4 - Beta" dependencies = [ "setuptools", + "db-dtypes >=0.3.0,<2.0.0", "numpy>=1.16.6", - "pandas>=0.23.2", + "pandas>=0.24.2", "pyarrow >=3.0.0, <7.0dev", "pydata-google-auth", "google-auth", @@ -35,7 +36,6 @@ ] extras = { "tqdm": "tqdm>=4.23.0", - "db-dtypes": "db-dtypes >=0.3.0,<2.0.0", } # Setup boilerplate below this line. diff --git a/packages/pandas-gbq/testing/constraints-3.7.txt b/packages/pandas-gbq/testing/constraints-3.7.txt index 7c67d2750fe2..7920656a9b12 100644 --- a/packages/pandas-gbq/testing/constraints-3.7.txt +++ b/packages/pandas-gbq/testing/constraints-3.7.txt @@ -5,12 +5,13 @@ # # e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", # Then this file should have foo==1.14.0 +db-dtypes==0.3.0 google-auth==1.4.1 google-auth-oauthlib==0.0.1 google-cloud-bigquery==1.11.1 google-cloud-bigquery-storage==1.1.0 numpy==1.16.6 -pandas==0.23.2 +pandas==0.24.2 pyarrow==3.0.0 pydata-google-auth==0.1.2 tqdm==4.23.0 diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index a8d6bd0ddc1b..f268a85d441a 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -26,8 +26,6 @@ TABLE_ID = "new_test" PANDAS_VERSION = pkg_resources.parse_version(pandas.__version__) -NULLABLE_INT_PANDAS_VERSION = pkg_resources.parse_version("0.24.0") -NULLABLE_INT_MESSAGE = "Require pandas 0.24+ in order to use nullable integer type." def test_imports(): @@ -173,9 +171,6 @@ def test_should_properly_handle_valid_integers(self, project_id): tm.assert_frame_equal(df, DataFrame({"valid_integer": [3]})) def test_should_properly_handle_nullable_integers(self, project_id): - if PANDAS_VERSION < NULLABLE_INT_PANDAS_VERSION: - pytest.skip(msg=NULLABLE_INT_MESSAGE) - query = """SELECT * FROM UNNEST([1, NULL]) AS nullable_integer """ @@ -188,9 +183,7 @@ def test_should_properly_handle_nullable_integers(self, project_id): ) tm.assert_frame_equal( df, - DataFrame( - {"nullable_integer": pandas.Series([1, pandas.NA], dtype="Int64")} - ), + DataFrame({"nullable_integer": pandas.Series([1, None], dtype="Int64")}), ) def test_should_properly_handle_valid_longs(self, project_id): @@ -204,9 +197,6 @@ def test_should_properly_handle_valid_longs(self, project_id): tm.assert_frame_equal(df, DataFrame({"valid_long": [1 << 62]})) def test_should_properly_handle_nullable_longs(self, project_id): - if PANDAS_VERSION < NULLABLE_INT_PANDAS_VERSION: - pytest.skip(msg=NULLABLE_INT_MESSAGE) - query = """SELECT * FROM UNNEST([1 << 62, NULL]) AS nullable_long """ @@ -219,15 +209,10 @@ def test_should_properly_handle_nullable_longs(self, project_id): ) tm.assert_frame_equal( df, - DataFrame( - {"nullable_long": pandas.Series([1 << 62, pandas.NA], dtype="Int64")} - ), + DataFrame({"nullable_long": pandas.Series([1 << 62, None], dtype="Int64")}), ) def test_should_properly_handle_null_integers(self, project_id): - if PANDAS_VERSION < NULLABLE_INT_PANDAS_VERSION: - pytest.skip(msg=NULLABLE_INT_MESSAGE) - query = "SELECT CAST(NULL AS INT64) AS null_integer" df = gbq.read_gbq( query, @@ -237,7 +222,7 @@ def test_should_properly_handle_null_integers(self, project_id): dtypes={"null_integer": "Int64"}, ) tm.assert_frame_equal( - df, DataFrame({"null_integer": pandas.Series([pandas.NA], dtype="Int64")}), + df, DataFrame({"null_integer": pandas.Series([None], dtype="Int64")}), ) def test_should_properly_handle_valid_floats(self, project_id): diff --git a/packages/pandas-gbq/tests/system/test_to_gbq.py b/packages/pandas-gbq/tests/system/test_to_gbq.py index 4f315a77aacd..4421f3be0f08 100644 --- a/packages/pandas-gbq/tests/system/test_to_gbq.py +++ b/packages/pandas-gbq/tests/system/test_to_gbq.py @@ -2,23 +2,22 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. +import datetime +import decimal +import collections import functools import random +import db_dtypes import pandas import pandas.testing import pytest -try: - import db_dtypes -except ImportError: - db_dtypes = None - pytest.importorskip("google.cloud.bigquery", minversion="1.24.0") -@pytest.fixture(params=["default", "load_parquet", "load_csv"]) +@pytest.fixture(params=["load_parquet", "load_csv"]) def api_method(request): return request.param @@ -32,13 +31,20 @@ def method_under_test(credentials, project_id): ) +SeriesRoundTripTestCase = collections.namedtuple( + "SeriesRoundTripTestCase", + ["input_series", "api_methods"], + defaults=[None, {"load_csv", "load_parquet"}], +) + + @pytest.mark.parametrize( - ["input_series", "skip_csv"], + ["input_series", "api_methods"], [ # Ensure that 64-bit floating point numbers are unchanged. # See: https://github.com/pydata/pandas-gbq/issues/326 - ( - pandas.Series( + SeriesRoundTripTestCase( + input_series=pandas.Series( [ 0.14285714285714285, 0.4406779661016949, @@ -51,10 +57,9 @@ def method_under_test(credentials, project_id): ], name="test_col", ), - False, ), - ( - pandas.Series( + SeriesRoundTripTestCase( + input_series=pandas.Series( [ "abc", "defg", @@ -66,10 +71,9 @@ def method_under_test(credentials, project_id): ], name="test_col", ), - False, ), - ( - pandas.Series( + SeriesRoundTripTestCase( + input_series=pandas.Series( [ "abc", "defg", @@ -81,7 +85,13 @@ def method_under_test(credentials, project_id): ], name="empty_strings", ), - True, + # BigQuery CSV loader uses empty string as the "null marker" by + # default. Potentially one could choose a rarely used character or + # string as the null marker to disambiguate null from empty string, + # but then that string couldn't be loaded. + # TODO: Revist when custom load job configuration is supported. + # https://github.com/googleapis/python-bigquery-pandas/issues/425 + api_methods={"load_parquet"}, ), ], ) @@ -91,10 +101,10 @@ def test_series_round_trip( bigquery_client, input_series, api_method, - skip_csv, + api_methods, ): - if api_method == "load_csv" and skip_csv: - pytest.skip("Loading with CSV not supported.") + if api_method not in api_methods: + pytest.skip(f"{api_method} not supported.") table_id = f"{random_dataset_id}.round_trip_{random.randrange(1_000_000)}" input_series = input_series.sort_values().reset_index(drop=True) df = pandas.DataFrame( @@ -111,60 +121,99 @@ def test_series_round_trip( ) +DataFrameRoundTripTestCase = collections.namedtuple( + "DataFrameRoundTripTestCase", + ["input_df", "expected_df", "table_schema", "api_methods"], + defaults=[None, None, [], {"load_csv", "load_parquet"}], +) + DATAFRAME_ROUND_TRIPS = [ # Ensure that a DATE column can be written with datetime64[ns] dtype # data. See: # https://github.com/googleapis/python-bigquery-pandas/issues/362 - ( - pandas.DataFrame( + DataFrameRoundTripTestCase( + input_df=pandas.DataFrame( { + "row_num": [0, 1, 2], "date_col": pandas.Series( ["2021-04-17", "1999-12-31", "2038-01-19"], dtype="datetime64[ns]", ), } ), - [{"name": "date_col", "type": "DATE"}], - True, + table_schema=[{"name": "date_col", "type": "DATE"}], + # Skip CSV because the pandas CSV writer includes time when writing + # datetime64 values. + api_methods={"load_parquet"}, + ), + DataFrameRoundTripTestCase( + input_df=pandas.DataFrame( + { + "row_num": [0, 1, 2], + "date_col": pandas.Series( + ["2021-04-17", "1999-12-31", "2038-01-19"], + dtype=db_dtypes.DateDtype(), + ), + } + ), + table_schema=[{"name": "date_col", "type": "DATE"}], + ), + # Loading a DATE column should work for string objects. See: + # https://github.com/googleapis/python-bigquery-pandas/issues/421 + DataFrameRoundTripTestCase( + input_df=pandas.DataFrame( + {"row_num": [123], "date_col": ["2021-12-12"]}, + columns=["row_num", "date_col"], + ), + expected_df=pandas.DataFrame( + {"row_num": [123], "date_col": [datetime.date(2021, 12, 12)]}, + columns=["row_num", "date_col"], + ), + table_schema=[ + {"name": "row_num", "type": "INTEGER"}, + {"name": "date_col", "type": "DATE"}, + ], + ), + # Loading a NUMERIC column should work for floating point objects. See: + # https://github.com/googleapis/python-bigquery-pandas/issues/421 + DataFrameRoundTripTestCase( + input_df=pandas.DataFrame( + {"row_num": [123], "num_col": [1.25]}, columns=["row_num", "num_col"], + ), + expected_df=pandas.DataFrame( + {"row_num": [123], "num_col": [decimal.Decimal("1.25")]}, + columns=["row_num", "num_col"], + ), + table_schema=[ + {"name": "row_num", "type": "INTEGER"}, + {"name": "num_col", "type": "NUMERIC"}, + ], ), ] -if db_dtypes is not None: - DATAFRAME_ROUND_TRIPS.append( - ( - pandas.DataFrame( - { - "date_col": pandas.Series( - ["2021-04-17", "1999-12-31", "2038-01-19"], dtype="dbdate", - ), - } - ), - [{"name": "date_col", "type": "DATE"}], - False, - ) - ) @pytest.mark.parametrize( - ["input_df", "table_schema", "skip_csv"], DATAFRAME_ROUND_TRIPS + ["input_df", "expected_df", "table_schema", "api_methods"], DATAFRAME_ROUND_TRIPS ) def test_dataframe_round_trip_with_table_schema( method_under_test, random_dataset_id, bigquery_client, input_df, + expected_df, table_schema, api_method, - skip_csv, + api_methods, ): - if api_method == "load_csv" and skip_csv: - pytest.skip("Loading with CSV not supported.") + if api_method not in api_methods: + pytest.skip(f"{api_method} not supported.") + if expected_df is None: + expected_df = input_df table_id = f"{random_dataset_id}.round_trip_w_schema_{random.randrange(1_000_000)}" - input_df["row_num"] = input_df.index - input_df.sort_values("row_num", inplace=True) method_under_test( input_df, table_id, table_schema=table_schema, api_method=api_method ) round_trip = bigquery_client.list_rows(table_id).to_dataframe( - dtypes=dict(zip(input_df.columns, input_df.dtypes)) + dtypes=dict(zip(expected_df.columns, expected_df.dtypes)) ) round_trip.sort_values("row_num", inplace=True) - pandas.testing.assert_frame_equal(input_df, round_trip) + pandas.testing.assert_frame_equal(expected_df, round_trip) diff --git a/packages/pandas-gbq/tests/unit/test_load.py b/packages/pandas-gbq/tests/unit/test_load.py index a32d2d9e62ea..8e18cfb9e5c7 100644 --- a/packages/pandas-gbq/tests/unit/test_load.py +++ b/packages/pandas-gbq/tests/unit/test_load.py @@ -4,12 +4,16 @@ # -*- coding: utf-8 -*- -import textwrap +import datetime +import decimal from io import StringIO +import textwrap from unittest import mock +import db_dtypes import numpy import pandas +import pandas.testing import pytest from pandas_gbq.features import FEATURES @@ -137,3 +141,117 @@ def test_load_chunks_omits_policy_tags( def test_load_chunks_with_invalid_api_method(): with pytest.raises(ValueError, match="Got unexpected api_method:"): load.load_chunks(None, None, None, api_method="not_a_thing") + + +@pytest.mark.parametrize( + ("numeric_type",), + ( + ("NUMERIC",), + ("DECIMAL",), + ("BIGNUMERIC",), + ("BIGDECIMAL",), + ("numeric",), + ("decimal",), + ("bignumeric",), + ("bigdecimal",), + ), +) +def test_cast_dataframe_for_parquet_w_float_numeric(numeric_type): + dataframe = pandas.DataFrame( + { + "row_num": [0, 1, 2], + "num_col": pandas.Series( + # Very much not recommend as the whole point of NUMERIC is to + # be more accurate than a floating point number, but tested to + # keep compatibility with CSV-based uploads. See: + # https://github.com/googleapis/python-bigquery-pandas/issues/421 + [1.25, -1.25, 42.5], + dtype="float64", + ), + "row_num_2": [0, 1, 2], + }, + # Use multiple columns to ensure column order is maintained. + columns=["row_num", "num_col", "row_num_2"], + ) + schema = { + "fields": [ + {"name": "num_col", "type": numeric_type}, + {"name": "not_in_df", "type": "IGNORED"}, + ] + } + result = load.cast_dataframe_for_parquet(dataframe, schema) + expected = pandas.DataFrame( + { + "row_num": [0, 1, 2], + "num_col": pandas.Series( + [decimal.Decimal(1.25), decimal.Decimal(-1.25), decimal.Decimal(42.5)], + dtype="object", + ), + "row_num_2": [0, 1, 2], + }, + columns=["row_num", "num_col", "row_num_2"], + ) + pandas.testing.assert_frame_equal(result, expected) + + +def test_cast_dataframe_for_parquet_w_string_date(): + dataframe = pandas.DataFrame( + { + "row_num": [0, 1, 2], + "date_col": pandas.Series( + ["2021-04-17", "1999-12-31", "2038-01-19"], dtype="object", + ), + "row_num_2": [0, 1, 2], + }, + # Use multiple columns to ensure column order is maintained. + columns=["row_num", "date_col", "row_num_2"], + ) + schema = { + "fields": [ + {"name": "date_col", "type": "DATE"}, + {"name": "not_in_df", "type": "IGNORED"}, + ] + } + result = load.cast_dataframe_for_parquet(dataframe, schema) + expected = pandas.DataFrame( + { + "row_num": [0, 1, 2], + "date_col": pandas.Series( + ["2021-04-17", "1999-12-31", "2038-01-19"], dtype=db_dtypes.DateDtype(), + ), + "row_num_2": [0, 1, 2], + }, + columns=["row_num", "date_col", "row_num_2"], + ) + pandas.testing.assert_frame_equal(result, expected) + + +def test_cast_dataframe_for_parquet_ignores_repeated_fields(): + dataframe = pandas.DataFrame( + { + "row_num": [0, 1, 2], + "repeated_col": pandas.Series( + [ + [datetime.date(2021, 4, 17)], + [datetime.date(199, 12, 31)], + [datetime.date(2038, 1, 19)], + ], + dtype="object", + ), + "row_num_2": [0, 1, 2], + }, + # Use multiple columns to ensure column order is maintained. + columns=["row_num", "repeated_col", "row_num_2"], + ) + expected = dataframe.copy() + schema = {"fields": [{"name": "repeated_col", "type": "DATE", "mode": "REPEATED"}]} + result = load.cast_dataframe_for_parquet(dataframe, schema) + pandas.testing.assert_frame_equal(result, expected) + + +def test_cast_dataframe_for_parquet_w_null_fields(): + dataframe = pandas.DataFrame({"int_col": [0, 1, 2], "str_col": ["a", "b", "c"]}) + expected = dataframe.copy() + schema = {"fields": None} + result = load.cast_dataframe_for_parquet(dataframe, schema) + pandas.testing.assert_frame_equal(result, expected) From d19bf81409f61066d0fe6b939067ef504ba96c05 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 30 Nov 2021 12:54:43 -0600 Subject: [PATCH 248/519] test: add session with prerelease dependencies (#428) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test: add session with prerelease dependencies * fix owlbot * add kokoro configs * match whitespace * remove extra whitespace * escape in replacement * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * move nox session to avoid duplicates * escape more parens * escape asterix * missing comma * don't remove indent Co-authored-by: Owl Bot --- .../.kokoro/continuous/prerelease.cfg | 7 ++ .../.kokoro/presubmit/prerelease.cfg | 7 ++ packages/pandas-gbq/noxfile.py | 84 ++++++++++++++++ packages/pandas-gbq/owlbot.py | 99 +++++++++++++++++++ 4 files changed, 197 insertions(+) create mode 100644 packages/pandas-gbq/.kokoro/continuous/prerelease.cfg create mode 100644 packages/pandas-gbq/.kokoro/presubmit/prerelease.cfg diff --git a/packages/pandas-gbq/.kokoro/continuous/prerelease.cfg b/packages/pandas-gbq/.kokoro/continuous/prerelease.cfg new file mode 100644 index 000000000000..00bc8678bc4a --- /dev/null +++ b/packages/pandas-gbq/.kokoro/continuous/prerelease.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "prerelease" +} diff --git a/packages/pandas-gbq/.kokoro/presubmit/prerelease.cfg b/packages/pandas-gbq/.kokoro/presubmit/prerelease.cfg new file mode 100644 index 000000000000..00bc8678bc4a --- /dev/null +++ b/packages/pandas-gbq/.kokoro/presubmit/prerelease.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "prerelease" +} diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 2feeccdc9f37..df3378bf181d 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -19,6 +19,7 @@ from __future__ import absolute_import import os import pathlib +import re import shutil import nox @@ -167,6 +168,89 @@ def system(session): ) +@nox.session(python=DEFAULT_PYTHON_VERSION) +def prerelease(session): + session.install( + "--extra-index-url", + "https://pypi.fury.io/arrow-nightlies/", + "--prefer-binary", + "--pre", + "--upgrade", + "pyarrow", + ) + session.install( + "--extra-index-url", + "https://pypi.anaconda.org/scipy-wheels-nightly/simple", + "--prefer-binary", + "--pre", + "--upgrade", + "pandas", + ) + session.install( + "--prefer-binary", + "--pre", + "--upgrade", + "google-api-core", + "google-cloud-bigquery", + "google-cloud-bigquery-storage", + "google-cloud-core", + "google-resumable-media", + "grpcio", + ) + session.install( + "freezegun", + "google-cloud-datacatalog", + "google-cloud-storage", + "google-cloud-testutils", + "IPython", + "mock", + "psutil", + "pytest", + "pytest-cov", + ) + + # Because we test minimum dependency versions on the minimum Python + # version, the first version we test with in the unit tests sessions has a + # constraints file containing all dependencies and extras. + with open( + CURRENT_DIRECTORY + / "testing" + / f"constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt", + encoding="utf-8", + ) as constraints_file: + constraints_text = constraints_file.read() + + # Ignore leading whitespace and comment lines. + deps = [ + match.group(1) + for match in re.finditer( + r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE + ) + ] + + # We use --no-deps to ensure that pre-release versions aren't overwritten + # by the version ranges in setup.py. + session.install(*deps) + session.install("--no-deps", "-e", ".[all]") + + # Print out prerelease package versions. + session.run("python", "-m", "pip", "freeze") + + # Run all tests, except a few samples tests which require extra dependencies. + session.run( + "py.test", + "--quiet", + f"--junitxml=prerelease_unit_{session.python}_sponge_log.xml", + os.path.join("tests", "unit"), + ) + session.run( + "py.test", + "--quiet", + f"--junitxml=prerelease_system_{session.python}_sponge_log.xml", + os.path.join("tests", "system"), + ) + + @nox.session(python=DEFAULT_PYTHON_VERSION) def cover(session): """Run the final coverage report. diff --git a/packages/pandas-gbq/owlbot.py b/packages/pandas-gbq/owlbot.py index c69d54deae56..5ef93de79e89 100644 --- a/packages/pandas-gbq/owlbot.py +++ b/packages/pandas-gbq/owlbot.py @@ -15,6 +15,7 @@ """This script is used to synthesize generated parts of this library.""" import pathlib +import re import synthtool as s from synthtool import gcp @@ -55,14 +56,112 @@ # Fixup files # ---------------------------------------------------------------------------- +s.replace( + ["noxfile.py"], + r"import pathlib\s+import shutil", + "import pathlib\nimport re\nimport shutil", +) + s.replace( ["noxfile.py"], r"[\"']google[\"']", '"pandas_gbq"', ) + s.replace( ["noxfile.py"], "--cov=google", "--cov=pandas_gbq", ) +s.replace( + ["noxfile.py"], + r"@nox.session\(python=DEFAULT_PYTHON_VERSION\)\s+def cover\(session\):", + r"""@nox.session(python=DEFAULT_PYTHON_VERSION) +def prerelease(session): + session.install( + "--extra-index-url", + "https://pypi.fury.io/arrow-nightlies/", + "--prefer-binary", + "--pre", + "--upgrade", + "pyarrow", + ) + session.install( + "--extra-index-url", + "https://pypi.anaconda.org/scipy-wheels-nightly/simple", + "--prefer-binary", + "--pre", + "--upgrade", + "pandas", + ) + session.install( + "--prefer-binary", + "--pre", + "--upgrade", + "google-api-core", + "google-cloud-bigquery", + "google-cloud-bigquery-storage", + "google-cloud-core", + "google-resumable-media", + "grpcio", + ) + session.install( + "freezegun", + "google-cloud-datacatalog", + "google-cloud-storage", + "google-cloud-testutils", + "IPython", + "mock", + "psutil", + "pytest", + "pytest-cov", + ) + + # Because we test minimum dependency versions on the minimum Python + # version, the first version we test with in the unit tests sessions has a + # constraints file containing all dependencies and extras. + with open( + CURRENT_DIRECTORY + / "testing" + / f"constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt", + encoding="utf-8", + ) as constraints_file: + constraints_text = constraints_file.read() + + # Ignore leading whitespace and comment lines. + deps = [ + match.group(1) + for match in re.finditer( + r"^\\s*(\\S+)(?===\\S+)", constraints_text, flags=re.MULTILINE + ) + ] + + # We use --no-deps to ensure that pre-release versions aren't overwritten + # by the version ranges in setup.py. + session.install(*deps) + session.install("--no-deps", "-e", ".[all]") + + # Print out prerelease package versions. + session.run("python", "-m", "pip", "freeze") + + # Run all tests, except a few samples tests which require extra dependencies. + session.run( + "py.test", + "--quiet", + f"--junitxml=prerelease_unit_{session.python}_sponge_log.xml", + os.path.join("tests", "unit"), + ) + session.run( + "py.test", + "--quiet", + f"--junitxml=prerelease_system_{session.python}_sponge_log.xml", + os.path.join("tests", "system"), + ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def cover(session):""", + re.MULTILINE, +) + s.replace( [".github/header-checker-lint.yml"], '"Google LLC"', '"pandas-gbq Authors"', ) From b2e329f8872837e0b0ce26d1fd76d0acd71c9937 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 7 Dec 2021 14:26:16 -0600 Subject: [PATCH 249/519] fix: allow extreme DATE values such as `datetime.date(1, 1, 1)` in `load_gbq` (#442) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - [ ] Make sure to open an issue as a [bug/issue](https://github.com/googleapis/python-bigquery-pandas/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea - [ ] Ensure the tests and linter pass - [ ] Code coverage does not decrease (if any source code was changed) - [ ] Appropriate docs were updated (if necessary) Fixes #441 Towards #365 🦕 --- .../ci/requirements-3.7-0.24.2.conda | 2 +- packages/pandas-gbq/pandas_gbq/load.py | 13 ++--- packages/pandas-gbq/setup.py | 8 ++-- .../pandas-gbq/testing/constraints-3.7.txt | 2 +- .../pandas-gbq/tests/system/test_to_gbq.py | 48 +++++++++++++++++++ 5 files changed, 61 insertions(+), 12 deletions(-) diff --git a/packages/pandas-gbq/ci/requirements-3.7-0.24.2.conda b/packages/pandas-gbq/ci/requirements-3.7-0.24.2.conda index 82f4e7b9a454..e0323d926b17 100644 --- a/packages/pandas-gbq/ci/requirements-3.7-0.24.2.conda +++ b/packages/pandas-gbq/ci/requirements-3.7-0.24.2.conda @@ -1,6 +1,6 @@ codecov coverage -db-dtypes==0.3.0 +db-dtypes==0.3.1 fastavro flake8 numpy==1.16.6 diff --git a/packages/pandas-gbq/pandas_gbq/load.py b/packages/pandas-gbq/pandas_gbq/load.py index 5422402e0e54..315ad5cdf217 100644 --- a/packages/pandas-gbq/pandas_gbq/load.py +++ b/packages/pandas-gbq/pandas_gbq/load.py @@ -90,12 +90,13 @@ def cast_dataframe_for_parquet( # Use extension dtype first so that it uses the correct equality operator. and db_dtypes.DateDtype() != dataframe[column_name].dtype ): - # Construct converted column manually, because I can't use - # .astype() with DateDtype. With .astype(), I get the error: - # - # TypeError: Cannot interpret '' as a data type - cast_column = pandas.Series( - dataframe[column_name], dtype=db_dtypes.DateDtype() + cast_column = dataframe[column_name].astype( + dtype=db_dtypes.DateDtype(), + # Return the original column if there was an error converting + # to the dtype, such as is there is a date outside the + # supported range. + # https://github.com/googleapis/python-bigquery-pandas/issues/441 + errors="ignore", ) elif column_type in {"NUMERIC", "DECIMAL", "BIGNUMERIC", "BIGDECIMAL"}: cast_column = dataframe[column_name].map(decimal.Decimal) diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index 28c81eee087f..283e5ea8a897 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -23,16 +23,16 @@ release_status = "Development Status :: 4 - Beta" dependencies = [ "setuptools", - "db-dtypes >=0.3.0,<2.0.0", - "numpy>=1.16.6", - "pandas>=0.24.2", + "db-dtypes >=0.3.1,<2.0.0", + "numpy >=1.16.6", + "pandas >=0.24.2", "pyarrow >=3.0.0, <7.0dev", "pydata-google-auth", "google-auth", "google-auth-oauthlib", # 2.4.* has a bug where waiting for the query can hang indefinitely. # https://github.com/pydata/pandas-gbq/issues/343 - "google-cloud-bigquery[bqstorage,pandas]>=1.11.1,<3.0.0dev,!=2.4.*", + "google-cloud-bigquery[bqstorage,pandas] >=1.11.1,<3.0.0dev,!=2.4.*", ] extras = { "tqdm": "tqdm>=4.23.0", diff --git a/packages/pandas-gbq/testing/constraints-3.7.txt b/packages/pandas-gbq/testing/constraints-3.7.txt index 7920656a9b12..6c3080dc63c9 100644 --- a/packages/pandas-gbq/testing/constraints-3.7.txt +++ b/packages/pandas-gbq/testing/constraints-3.7.txt @@ -5,7 +5,7 @@ # # e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", # Then this file should have foo==1.14.0 -db-dtypes==0.3.0 +db-dtypes==0.3.1 google-auth==1.4.1 google-auth-oauthlib==0.0.1 google-cloud-bigquery==1.11.1 diff --git a/packages/pandas-gbq/tests/system/test_to_gbq.py b/packages/pandas-gbq/tests/system/test_to_gbq.py index 4421f3be0f08..f718402425d5 100644 --- a/packages/pandas-gbq/tests/system/test_to_gbq.py +++ b/packages/pandas-gbq/tests/system/test_to_gbq.py @@ -188,6 +188,54 @@ def test_series_round_trip( {"name": "num_col", "type": "NUMERIC"}, ], ), + pytest.param( + *DataFrameRoundTripTestCase( + input_df=pandas.DataFrame( + { + "row_num": [1, 2, 3], + # DATE valuess outside the pandas range for timestamp + # aren't supported by the db-dtypes package. + # https://github.com/googleapis/python-bigquery-pandas/issues/441 + "date_col": [ + datetime.date(1, 1, 1), + datetime.date(1970, 1, 1), + datetime.date(9999, 12, 31), + ], + # TODO: DATETIME/TIMESTAMP values outside of the range for + # pandas timestamp require `date_as_object` parameter in + # google-cloud-bigquery versions 1.x and 2.x. + # https://github.com/googleapis/python-bigquery-pandas/issues/365 + # "datetime_col": [ + # datetime.datetime(1, 1, 1), + # datetime.datetime(1970, 1, 1), + # datetime.datetime(9999, 12, 31, 23, 59, 59, 999999), + # ], + # "timestamp_col": [ + # datetime.datetime(1, 1, 1, tzinfo=datetime.timezone.utc), + # datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc), + # datetime.datetime( + # 9999, + # 12, + # 31, + # 23, + # 59, + # 59, + # 999999, + # tzinfo=datetime.timezone.utc, + # ), + # ], + }, + columns=["row_num", "date_col", "datetime_col", "timestamp_col"], + ), + table_schema=[ + {"name": "row_num", "type": "INTEGER"}, + {"name": "date_col", "type": "DATE"}, + {"name": "datetime_col", "type": "DATETIME"}, + {"name": "timestamp_col", "type": "TIMESTAMP"}, + ], + ), + id="issue365-extreme-datetimes", + ), ] From 732b4deb86990f47ed88ebbff9e89360979efd3e Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 9 Dec 2021 09:27:16 -0600 Subject: [PATCH 250/519] feat!: use nullable Int64 and boolean dtypes if available (#445) * feat: use nullable Int64 and boolean dtypes if available * allow google-cloud-bigquery 3.x * document dtypes mapping --- packages/pandas-gbq/docs/reading.rst | 8 ++- packages/pandas-gbq/pandas_gbq/features.py | 8 +++ packages/pandas-gbq/pandas_gbq/gbq.py | 7 +- packages/pandas-gbq/setup.py | 2 +- packages/pandas-gbq/tests/system/test_gbq.py | 70 ++++++++++++++------ packages/pandas-gbq/tests/unit/test_gbq.py | 4 +- 6 files changed, 73 insertions(+), 26 deletions(-) diff --git a/packages/pandas-gbq/docs/reading.rst b/packages/pandas-gbq/docs/reading.rst index aaecf9a0f188..e3e3dc5a20bd 100644 --- a/packages/pandas-gbq/docs/reading.rst +++ b/packages/pandas-gbq/docs/reading.rst @@ -59,11 +59,13 @@ column, based on the BigQuery table schema. ================== ========================= BigQuery Data Type dtype ================== ========================= -FLOAT float -TIMESTAMP :class:`~pandas.DatetimeTZDtype` with ``unit='ns'`` and ``tz='UTC'`` +DATE datetime64[ns] DATETIME datetime64[ns] +BOOL boolean +FLOAT float +INT64 Int64 TIME datetime64[ns] -DATE datetime64[ns] +TIMESTAMP :class:`~pandas.DatetimeTZDtype` with ``unit='ns'`` and ``tz='UTC'`` ================== ========================= .. _reading-bqstorage-api: diff --git a/packages/pandas-gbq/pandas_gbq/features.py b/packages/pandas-gbq/pandas_gbq/features.py index fc8ef5684548..4259eaf12198 100644 --- a/packages/pandas-gbq/pandas_gbq/features.py +++ b/packages/pandas-gbq/pandas_gbq/features.py @@ -10,6 +10,7 @@ BIGQUERY_BQSTORAGE_VERSION = "1.24.0" BIGQUERY_FROM_DATAFRAME_CSV_VERSION = "2.6.0" PANDAS_VERBOSITY_DEPRECATION_VERSION = "0.23.0" +PANDAS_BOOLEAN_DTYPE_VERSION = "1.0.0" PANDAS_PARQUET_LOSSLESS_TIMESTAMP_VERSION = "1.1.0" @@ -90,6 +91,13 @@ def pandas_has_deprecated_verbose(self): ) return self.pandas_installed_version >= pandas_verbosity_deprecation + @property + def pandas_has_boolean_dtype(self): + import pkg_resources + + desired_version = pkg_resources.parse_version(PANDAS_BOOLEAN_DTYPE_VERSION) + return self.pandas_installed_version >= desired_version + @property def pandas_has_parquet_with_lossless_timestamp(self): import pkg_resources diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 87c2327c9ccc..a1ae289612a7 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -579,12 +579,13 @@ def _bqschema_to_nullsafe_dtypes(schema_fields): #missing-data-casting-rules-and-indexing """ # If you update this mapping, also update the table at - # `docs/source/reading.rst`. + # `docs/reading.rst`. dtype_map = { "DATE": "datetime64[ns]", "DATETIME": "datetime64[ns]", "FLOAT": np.dtype(float), "GEOMETRY": "object", + "INTEGER": "Int64", "RECORD": "object", "STRING": "object", # datetime.time objects cannot be case to datetime64. @@ -596,6 +597,10 @@ def _bqschema_to_nullsafe_dtypes(schema_fields): "TIMESTAMP": "datetime64[ns]", } + # Amend dtype_map with newer extension types if pandas version allows. + if FEATURES.pandas_has_boolean_dtype: + dtype_map["BOOLEAN"] = "boolean" + dtypes = {} for field in schema_fields: name = str(field["name"]) diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index 283e5ea8a897..4be5a72288e8 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -32,7 +32,7 @@ "google-auth-oauthlib", # 2.4.* has a bug where waiting for the query can hang indefinitely. # https://github.com/pydata/pandas-gbq/issues/343 - "google-cloud-bigquery[bqstorage,pandas] >=1.11.1,<3.0.0dev,!=2.4.*", + "google-cloud-bigquery[bqstorage,pandas] >=1.11.1,<4.0.0dev,!=2.4.*", ] extras = { "tqdm": "tqdm>=4.23.0", diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index f268a85d441a..812d2089454b 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -10,7 +10,7 @@ import numpy as np import pandas import pandas.api.types -import pandas.util.testing as tm +import pandas.testing as tm from pandas import DataFrame, NaT try: @@ -21,6 +21,7 @@ import pytz from pandas_gbq import gbq +from pandas_gbq.features import FEATURES import pandas_gbq.schema @@ -32,6 +33,18 @@ def test_imports(): gbq._test_google_api_imports() +def make_mixed_dataframe_v1(): + # Re-implementation of private pandas.util.testing.makeMixedDataFrame + return pandas.DataFrame( + { + "A": [0.0, 1.0, 2.0, 3.0, 4.0], + "B": [0.0, 1.0, 0.0, 1.0, 0.0], + "C": ["foo1", "foo2", "foo3", "foo4", "foo5"], + "D": pandas.bdate_range("1/1/2009", periods=5), + } + ) + + def make_mixed_dataframe_v2(test_size): # create df to test for all BQ datatypes except RECORD bools = np.random.randint(2, size=(1, test_size)).astype(bool) @@ -168,7 +181,7 @@ def test_should_properly_handle_valid_integers(self, project_id): credentials=self.credentials, dialect="standard", ) - tm.assert_frame_equal(df, DataFrame({"valid_integer": [3]})) + tm.assert_frame_equal(df, DataFrame({"valid_integer": [3]}, dtype="Int64")) def test_should_properly_handle_nullable_integers(self, project_id): query = """SELECT * FROM @@ -194,7 +207,7 @@ def test_should_properly_handle_valid_longs(self, project_id): credentials=self.credentials, dialect="standard", ) - tm.assert_frame_equal(df, DataFrame({"valid_long": [1 << 62]})) + tm.assert_frame_equal(df, DataFrame({"valid_long": [1 << 62]}, dtype="Int64")) def test_should_properly_handle_nullable_longs(self, project_id): query = """SELECT * FROM @@ -433,7 +446,10 @@ def test_should_properly_handle_null_boolean(self, project_id): credentials=self.credentials, dialect="legacy", ) - tm.assert_frame_equal(df, DataFrame({"null_boolean": [None]})) + expected_dtype = "boolean" if FEATURES.pandas_has_boolean_dtype else None + tm.assert_frame_equal( + df, DataFrame({"null_boolean": [None]}, dtype=expected_dtype) + ) def test_should_properly_handle_nullable_booleans(self, project_id): query = """SELECT * FROM @@ -445,8 +461,9 @@ def test_should_properly_handle_nullable_booleans(self, project_id): credentials=self.credentials, dialect="legacy", ) + expected_dtype = "boolean" if FEATURES.pandas_has_boolean_dtype else None tm.assert_frame_equal( - df, DataFrame({"nullable_boolean": [True, None]}).astype(object) + df, DataFrame({"nullable_boolean": [True, None]}, dtype=expected_dtype) ) def test_unicode_string_conversion_and_normalization(self, project_id): @@ -629,7 +646,7 @@ def test_one_row_one_column(self, project_id): credentials=self.credentials, dialect="standard", ) - expected_result = DataFrame(dict(v=[3])) + expected_result = DataFrame(dict(v=[3]), dtype="Int64") tm.assert_frame_equal(df, expected_result) def test_legacy_sql(self, project_id): @@ -719,7 +736,7 @@ def test_query_with_parameters(self, project_id): configuration=config, dialect="legacy", ) - tm.assert_frame_equal(df, DataFrame({"valid_result": [3]})) + tm.assert_frame_equal(df, DataFrame({"valid_result": [3]}, dtype="Int64")) def test_query_inside_configuration(self, project_id): query_no_use = 'SELECT "PI_WRONG" AS valid_string' @@ -842,7 +859,11 @@ def test_struct(self, project_id): dialect="standard", ) expected = DataFrame( - [[1, {"letter": "a", "num": 1}]], columns=["int_field", "struct_field"], + { + "int_field": pandas.Series([1], dtype="Int64"), + "struct_field": [{"letter": "a", "num": 1}], + }, + columns=["int_field", "struct_field"], ) tm.assert_frame_equal(df, expected) @@ -874,7 +895,12 @@ def test_array_length_zero(self, project_id): dialect="standard", ) expected = DataFrame( - [["a", [""], 1], ["b", [], 0]], columns=["letter", "array_field", "len"], + { + "letter": ["a", "b"], + "array_field": [[""], []], + "len": pandas.Series([1, 0], dtype="Int64"), + }, + columns=["letter", "array_field", "len"], ) tm.assert_frame_equal(df, expected) @@ -908,7 +934,13 @@ def test_array_of_floats(self, project_id): credentials=self.credentials, dialect="standard", ) - tm.assert_frame_equal(df, DataFrame([[[1.1, 2.2, 3.3], 4]], columns=["a", "b"])) + tm.assert_frame_equal( + df, + DataFrame( + {"a": [[1.1, 2.2, 3.3]], "b": pandas.Series([4], dtype="Int64")}, + columns=["a", "b"], + ), + ) def test_tokyo(self, tokyo_dataset, tokyo_table, project_id): df = gbq.read_gbq( @@ -1021,7 +1053,7 @@ def test_upload_data_if_table_exists_append(self, project_id): test_id = "3" test_size = 10 df = make_mixed_dataframe_v2(test_size) - df_different_schema = tm.makeMixedDataFrame() + df_different_schema = make_mixed_dataframe_v1() # Initialize table with sample data gbq.to_gbq( @@ -1101,7 +1133,7 @@ def test_upload_data_if_table_exists_replace(self, project_id): test_id = "4" test_size = 10 df = make_mixed_dataframe_v2(test_size) - df_different_schema = tm.makeMixedDataFrame() + df_different_schema = make_mixed_dataframe_v1() # Initialize table with sample data gbq.to_gbq( @@ -1225,7 +1257,7 @@ def test_upload_data_with_newlines(self, project_id): result = result_df["s"].sort_values() expected = df["s"].sort_values() - tm.assert_numpy_array_equal(expected.values, result.values) + tm.assert_series_equal(expected, result) def test_upload_data_flexible_column_order(self, project_id): test_id = "13" @@ -1254,7 +1286,7 @@ def test_upload_data_flexible_column_order(self, project_id): def test_upload_data_with_valid_user_schema(self, project_id): # Issue #46; tests test scenarios with user-provided # schemas - df = tm.makeMixedDataFrame() + df = make_mixed_dataframe_v1() test_id = "18" test_schema = [ {"name": "A", "type": "FLOAT"}, @@ -1276,7 +1308,7 @@ def test_upload_data_with_valid_user_schema(self, project_id): ) def test_upload_data_with_invalid_user_schema_raises_error(self, project_id): - df = tm.makeMixedDataFrame() + df = make_mixed_dataframe_v1() test_id = "19" test_schema = [ {"name": "A", "type": "FLOAT"}, @@ -1295,7 +1327,7 @@ def test_upload_data_with_invalid_user_schema_raises_error(self, project_id): ) def test_upload_data_with_missing_schema_fields_raises_error(self, project_id): - df = tm.makeMixedDataFrame() + df = make_mixed_dataframe_v1() test_id = "20" test_schema = [ {"name": "A", "type": "FLOAT"}, @@ -1351,7 +1383,7 @@ def test_upload_data_with_timestamp(self, project_id): tm.assert_series_equal(expected, result) def test_upload_data_with_different_df_and_user_schema(self, project_id): - df = tm.makeMixedDataFrame() + df = make_mixed_dataframe_v1() df["A"] = df["A"].astype(str) df["B"] = df["B"].astype(str) test_id = "22" @@ -1460,13 +1492,13 @@ def test_dataset_does_not_exist(gbq_dataset, random_dataset_id): def test_create_table(gbq_table): - schema = gbq._generate_bq_schema(tm.makeMixedDataFrame()) + schema = gbq._generate_bq_schema(make_mixed_dataframe_v1()) gbq_table.create("test_create_table", schema) assert gbq_table.exists("test_create_table") def test_create_table_already_exists(gbq_table): - schema = gbq._generate_bq_schema(tm.makeMixedDataFrame()) + schema = gbq._generate_bq_schema(make_mixed_dataframe_v1()) gbq_table.create("test_create_table_exists", schema) with pytest.raises(gbq.TableCreationError): gbq_table.create("test_create_table_exists", schema) diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index 0a5ecad24532..8784a98b4ae6 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -64,8 +64,8 @@ def no_auth(monkeypatch): @pytest.mark.parametrize( ("type_", "expected"), [ - ("INTEGER", None), # Can't handle NULL - ("BOOLEAN", None), # Can't handle NULL + ("SOME_NEW_UNKNOWN_TYPE", None), + ("INTEGER", "Int64"), ("FLOAT", numpy.dtype(float)), # TIMESTAMP will be localized after DataFrame construction. ("TIMESTAMP", "datetime64[ns]"), From 9dbc6ffd43808bc65ebd08477e6b133605c3f45c Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 9 Dec 2021 12:10:14 -0600 Subject: [PATCH 251/519] test: remove redundant NIGHTLY run from CircleCI (#446) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Eventually we will want to migrate these to Kokoro (or even GitHub actions using https://github.com/google-github-actions/auth) but just remove the broken session for now. The prerelease run is redundant with the session we recently added, so replace it with a session using the latest released pandas via conda. The reason we're not removing the conda sessions completely is that pandas recommends installation via conda/mamba. Since pandas-gbq is an optional dependency of pandas, we need to ensure this package is also always installable via conda/mamba. Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - [x] Make sure to open an issue as a [bug/issue](https://github.com/googleapis/python-bigquery-pandas/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea - [x] Ensure the tests and linter pass - [x] Code coverage does not decrease (if any source code was changed) - [x] Appropriate docs were updated (if necessary) Fixes #424 🦕 --- packages/pandas-gbq/.circleci/config.yml | 10 ++++---- ...TLY.conda => requirements-3.9-1.3.4.conda} | 11 +++++--- packages/pandas-gbq/ci/run_conda.sh | 25 ++++++------------- 3 files changed, 20 insertions(+), 26 deletions(-) rename packages/pandas-gbq/ci/{requirements-3.9-NIGHTLY.conda => requirements-3.9-1.3.4.conda} (86%) diff --git a/packages/pandas-gbq/.circleci/config.yml b/packages/pandas-gbq/.circleci/config.yml index 4c378b3fb758..e008054c67ad 100644 --- a/packages/pandas-gbq/.circleci/config.yml +++ b/packages/pandas-gbq/.circleci/config.yml @@ -7,7 +7,7 @@ jobs: # Conda "conda-3.7": docker: - - image: continuumio/miniconda3 + - image: mambaorg/micromamba environment: PYTHON: "3.7" PANDAS: "0.24.2" @@ -15,12 +15,12 @@ jobs: - checkout - run: ci/config_auth.sh - run: ci/run_conda.sh - "conda-3.9-NIGHTLY": + "conda-3.9": docker: - - image: continuumio/miniconda3 + - image: mambaorg/micromamba environment: PYTHON: "3.9" - PANDAS: "NIGHTLY" + PANDAS: "1.3.4" steps: - checkout - run: ci/config_auth.sh @@ -31,4 +31,4 @@ workflows: build: jobs: - "conda-3.7" - - "conda-3.9-NIGHTLY" + - "conda-3.9" diff --git a/packages/pandas-gbq/ci/requirements-3.9-NIGHTLY.conda b/packages/pandas-gbq/ci/requirements-3.9-1.3.4.conda similarity index 86% rename from packages/pandas-gbq/ci/requirements-3.9-NIGHTLY.conda rename to packages/pandas-gbq/ci/requirements-3.9-1.3.4.conda index 5a3e9fb72d21..73595253bdd6 100644 --- a/packages/pandas-gbq/ci/requirements-3.9-NIGHTLY.conda +++ b/packages/pandas-gbq/ci/requirements-3.9-1.3.4.conda @@ -1,10 +1,13 @@ +codecov +coverage db-dtypes -pydata-google-auth +fastavro +flake8 google-cloud-bigquery google-cloud-bigquery-storage +numpy pyarrow +pydata-google-auth pytest pytest-cov -codecov -coverage -flake8 +tqdm diff --git a/packages/pandas-gbq/ci/run_conda.sh b/packages/pandas-gbq/ci/run_conda.sh index e29da98aff86..11b5b569fff2 100755 --- a/packages/pandas-gbq/ci/run_conda.sh +++ b/packages/pandas-gbq/ci/run_conda.sh @@ -6,25 +6,16 @@ set -e -x DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" -# Install dependencies using Conda +eval "$(micromamba shell hook --shell=bash)" +micromamba activate -conda config --set always_yes yes --set changeps1 no -conda config --add channels pandas -conda config --add channels conda-forge -conda update -q conda -conda info -a -conda create -q -n test-environment python=$PYTHON -source activate test-environment +# Install dependencies using (micro)mamba +# https://github.com/mamba-org/micromamba-docker REQ="ci/requirements-${PYTHON}-${PANDAS}" -conda install -q --file "$REQ.conda"; - -if [[ "$PANDAS" == "NIGHTLY" ]]; then - conda install -q numpy pytz python-dateutil; - PRE_WHEELS="https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com"; - pip install --pre --upgrade --timeout=60 -f $PRE_WHEELS pandas; -else - conda install -q pandas=$PANDAS; -fi +micromamba install -q pandas=$PANDAS python=${PYTHON} -c conda-forge; +micromamba install -q --file "$REQ.conda" -c conda-forge; +micromamba list +micromamba info python setup.py develop --no-deps From 4dc6ff74695f480da12b3e840e38ffca80c9493e Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 22 Dec 2021 13:06:18 -0600 Subject: [PATCH 252/519] feat: accepts a table ID, which downloads the table without a query (#443) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - [x] Make sure to open an issue as a [bug/issue](https://github.com/googleapis/python-bigquery-pandas/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea - [x] Ensure the tests and linter pass - [x] Code coverage does not decrease (if any source code was changed) - [x] Appropriate docs were updated (if necessary) Fixes #266 🦕 --- packages/pandas-gbq/.coveragerc | 2 +- packages/pandas-gbq/noxfile.py | 2 +- packages/pandas-gbq/owlbot.py | 2 +- packages/pandas-gbq/pandas_gbq/gbq.py | 148 ++++++++++++------ packages/pandas-gbq/pandas_gbq/timestamp.py | 8 +- packages/pandas-gbq/tests/system/conftest.py | 19 +++ .../pandas-gbq/tests/system/test_to_gbq.py | 19 +-- packages/pandas-gbq/tests/unit/conftest.py | 23 ++- packages/pandas-gbq/tests/unit/test_gbq.py | 95 ++++++++++- 9 files changed, 245 insertions(+), 73 deletions(-) diff --git a/packages/pandas-gbq/.coveragerc b/packages/pandas-gbq/.coveragerc index ba50bf322ae0..88b85d0364aa 100644 --- a/packages/pandas-gbq/.coveragerc +++ b/packages/pandas-gbq/.coveragerc @@ -22,7 +22,7 @@ omit = google/cloud/__init__.py [report] -fail_under = 88 +fail_under = 89 show_missing = True exclude_lines = # Re-enable the standard pragma diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index df3378bf181d..398b4dc26baf 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -259,7 +259,7 @@ def cover(session): test runs (not system test runs), and then erases coverage data. """ session.install("coverage", "pytest-cov") - session.run("coverage", "report", "--show-missing", "--fail-under=88") + session.run("coverage", "report", "--show-missing", "--fail-under=89") session.run("coverage", "erase") diff --git a/packages/pandas-gbq/owlbot.py b/packages/pandas-gbq/owlbot.py index 5ef93de79e89..9849f98f4613 100644 --- a/packages/pandas-gbq/owlbot.py +++ b/packages/pandas-gbq/owlbot.py @@ -33,7 +33,7 @@ templated_files = common.py_library( unit_test_python_versions=["3.7", "3.8", "3.9", "3.10"], system_test_python_versions=["3.7", "3.8", "3.9", "3.10"], - cov_level=88, + cov_level=89, unit_test_extras=extras, system_test_extras=extras, intersphinx_dependencies={ diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index a1ae289612a7..5dcc3fd0f672 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -3,12 +3,21 @@ # license that can be found in the LICENSE file. import logging +import re import time import warnings from datetime import datetime +import typing +from typing import Any, Dict, Optional, Union import numpy as np +# Only import at module-level at type checking time to avoid circular +# dependencies in the pandas package, which has an optional dependency on +# pandas-gbq. +if typing.TYPE_CHECKING: # pragma: NO COVER + import pandas + # Required dependencies, but treat as optional so that _test_google_api_imports # can provide a better error message. try: @@ -64,6 +73,10 @@ def _test_google_api_imports(): raise ImportError("pandas-gbq requires google-cloud-bigquery") from ex +def _is_query(query_or_table: str) -> bool: + return re.search(r"\s", query_or_table.strip(), re.MULTILINE) is not None + + class DatasetCreationError(ValueError): """ Raised when the create dataset method fails @@ -374,6 +387,30 @@ def process_http_error(ex): raise GenericGBQException("Reason: {0}".format(ex)) + def download_table( + self, + table_id: str, + max_results: Optional[int] = None, + progress_bar_type: Optional[str] = None, + dtypes: Optional[Dict[str, Union[str, Any]]] = None, + ) -> "pandas.DataFrame": + self._start_timer() + + try: + table_ref = bigquery.TableReference.from_string( + table_id, default_project=self.project_id + ) + rows_iter = self.client.list_rows(table_ref, max_results=max_results) + except self.http_error as ex: + self.process_http_error(ex) + + return self._download_results( + rows_iter, + max_results=max_results, + progress_bar_type=progress_bar_type, + user_dtypes=dtypes, + ) + def run_query(self, query, max_results=None, progress_bar_type=None, **kwargs): from concurrent.futures import TimeoutError from google.auth.exceptions import RefreshError @@ -390,15 +427,6 @@ def run_query(self, query, max_results=None, progress_bar_type=None, **kwargs): if config is not None: job_config.update(config) - if "query" in config and "query" in config["query"]: - if query is not None: - raise ValueError( - "Query statement can't be specified " - "inside config while it is specified " - "as parameter" - ) - query = config["query"].pop("query") - self._start_timer() try: @@ -464,15 +492,25 @@ def run_query(self, query, max_results=None, progress_bar_type=None, **kwargs): ) dtypes = kwargs.get("dtypes") + + # Ensure destination is populated. + try: + query_reply.result() + except self.http_error as ex: + self.process_http_error(ex) + + rows_iter = self.client.list_rows( + query_reply.destination, max_results=max_results + ) return self._download_results( - query_reply, + rows_iter, max_results=max_results, progress_bar_type=progress_bar_type, user_dtypes=dtypes, ) def _download_results( - self, query_job, max_results=None, progress_bar_type=None, user_dtypes=None, + self, rows_iter, max_results=None, progress_bar_type=None, user_dtypes=None, ): # No results are desired, so don't bother downloading anything. if max_results == 0: @@ -504,11 +542,6 @@ def _download_results( to_dataframe_kwargs["create_bqstorage_client"] = create_bqstorage_client try: - query_job.result() - # Get the table schema, so that we can list rows. - destination = self.client.get_table(query_job.destination) - rows_iter = self.client.list_rows(destination, max_results=max_results) - schema_fields = [field.to_api_repr() for field in rows_iter.schema] conversion_dtypes = _bqschema_to_nullsafe_dtypes(schema_fields) conversion_dtypes.update(user_dtypes) @@ -644,7 +677,7 @@ def _cast_empty_df_dtypes(schema_fields, df): def read_gbq( - query, + query_or_table, project_id=None, index_col=None, col_order=None, @@ -668,17 +701,18 @@ def read_gbq( This method uses the Google Cloud client library to make requests to Google BigQuery, documented `here - `__. + `__. See the :ref:`How to authenticate with Google BigQuery ` guide for authentication instructions. Parameters ---------- - query : str - SQL-Like Query to return data values. + query_or_table : str + SQL query to return data values. If the string is a table ID, fetch the + rows directly from the table without running a query. project_id : str, optional - Google BigQuery Account project ID. Optional when available from + Google Cloud Platform project ID. Optional when available from the environment. index_col : str, optional Name of result column to use for index in results DataFrame. @@ -688,14 +722,14 @@ def read_gbq( reauth : boolean, default False Force Google BigQuery to re-authenticate the user. This is useful if multiple accounts are used. - auth_local_webserver : boolean, default False - Use the `local webserver flow`_ instead of the `console flow`_ - when getting user credentials. - - .. _local webserver flow: - http://google-auth-oauthlib.readthedocs.io/en/latest/reference/google_auth_oauthlib.flow.html#google_auth_oauthlib.flow.InstalledAppFlow.run_local_server - .. _console flow: - http://google-auth-oauthlib.readthedocs.io/en/latest/reference/google_auth_oauthlib.flow.html#google_auth_oauthlib.flow.InstalledAppFlow.run_console + auth_local_webserver : bool, default False + Use the `local webserver flow + `_ + instead of the `console flow + `_ + when getting user credentials. Your code must run on the same machine + as your web browser and your web browser can access your application + via ``localhost:808X``. .. versionadded:: 0.2.0 dialect : str, default 'standard' @@ -745,13 +779,6 @@ def read_gbq( `__ permission on the project you are billing queries to. - **Note:** Due to a `known issue in the ``google-cloud-bigquery`` - package - `__ - (fixed in version 1.11.0), you must write your query results to a - destination table. To do this with ``read_gbq``, supply a - ``configuration`` dictionary. - This feature requires the ``google-cloud-bigquery-storage`` and ``pyarrow`` packages. @@ -823,6 +850,15 @@ def read_gbq( if dialect not in ("legacy", "standard"): raise ValueError("'{0}' is not valid for dialect".format(dialect)) + if configuration and "query" in configuration and "query" in configuration["query"]: + if query_or_table is not None: + raise ValueError( + "Query statement can't be specified " + "inside config while it is specified " + "as parameter" + ) + query_or_table = configuration["query"].pop("query") + connector = GbqConnector( project_id, reauth=reauth, @@ -834,13 +870,21 @@ def read_gbq( use_bqstorage_api=use_bqstorage_api, ) - final_df = connector.run_query( - query, - configuration=configuration, - max_results=max_results, - progress_bar_type=progress_bar_type, - dtypes=dtypes, - ) + if _is_query(query_or_table): + final_df = connector.run_query( + query_or_table, + configuration=configuration, + max_results=max_results, + progress_bar_type=progress_bar_type, + dtypes=dtypes, + ) + else: + final_df = connector.download_table( + query_or_table, + max_results=max_results, + progress_bar_type=progress_bar_type, + dtypes=dtypes, + ) # Reindex the DataFrame on the provided column if index_col is not None: @@ -889,7 +933,7 @@ def to_gbq( This method uses the Google Cloud client library to make requests to Google BigQuery, documented `here - `__. + `__. See the :ref:`How to authenticate with Google BigQuery ` guide for authentication instructions. @@ -902,7 +946,7 @@ def to_gbq( Name of table to be written, in the form ``dataset.tablename`` or ``project.dataset.tablename``. project_id : str, optional - Google BigQuery Account project ID. Optional when available from + Google Cloud Platform project ID. Optional when available from the environment. chunksize : int, optional Number of rows to be inserted in each chunk from the dataframe. @@ -920,13 +964,13 @@ def to_gbq( ``'append'`` If table exists, insert data. Create if does not exist. auth_local_webserver : bool, default False - Use the `local webserver flow`_ instead of the `console flow`_ - when getting user credentials. - - .. _local webserver flow: - http://google-auth-oauthlib.readthedocs.io/en/latest/reference/google_auth_oauthlib.flow.html#google_auth_oauthlib.flow.InstalledAppFlow.run_local_server - .. _console flow: - http://google-auth-oauthlib.readthedocs.io/en/latest/reference/google_auth_oauthlib.flow.html#google_auth_oauthlib.flow.InstalledAppFlow.run_console + Use the `local webserver flow + `_ + instead of the `console flow + `_ + when getting user credentials. Your code must run on the same machine + as your web browser and your web browser can access your application + via ``localhost:808X``. .. versionadded:: 0.2.0 table_schema : list of dicts, optional diff --git a/packages/pandas-gbq/pandas_gbq/timestamp.py b/packages/pandas-gbq/pandas_gbq/timestamp.py index e0b414759a02..c6bb6d93286b 100644 --- a/packages/pandas-gbq/pandas_gbq/timestamp.py +++ b/packages/pandas-gbq/pandas_gbq/timestamp.py @@ -7,6 +7,8 @@ Private module. """ +import pandas.api.types + def localize_df(df, schema_fields): """Localize any TIMESTAMP columns to tz-aware type. @@ -38,7 +40,11 @@ def localize_df(df, schema_fields): if "mode" in field and field["mode"].upper() == "REPEATED": continue - if field["type"].upper() == "TIMESTAMP" and df[column].dt.tz is None: + if ( + field["type"].upper() == "TIMESTAMP" + and pandas.api.types.is_datetime64_ns_dtype(df.dtypes[column]) + and df[column].dt.tz is None + ): df[column] = df[column].dt.tz_localize("UTC") return df diff --git a/packages/pandas-gbq/tests/system/conftest.py b/packages/pandas-gbq/tests/system/conftest.py index 6ac55220262a..4ba8bf31008f 100644 --- a/packages/pandas-gbq/tests/system/conftest.py +++ b/packages/pandas-gbq/tests/system/conftest.py @@ -3,6 +3,7 @@ # license that can be found in the LICENSE file. import os +import functools import pathlib from google.cloud import bigquery @@ -56,6 +57,24 @@ def project(project_id): return project_id +@pytest.fixture +def to_gbq(credentials, project_id): + import pandas_gbq + + return functools.partial( + pandas_gbq.to_gbq, project_id=project_id, credentials=credentials + ) + + +@pytest.fixture +def read_gbq(credentials, project_id): + import pandas_gbq + + return functools.partial( + pandas_gbq.read_gbq, project_id=project_id, credentials=credentials + ) + + @pytest.fixture() def random_dataset_id(bigquery_client: bigquery.Client, project_id: str): dataset_id = prefixer.create_prefix() diff --git a/packages/pandas-gbq/tests/system/test_to_gbq.py b/packages/pandas-gbq/tests/system/test_to_gbq.py index f718402425d5..a9274091e8d7 100644 --- a/packages/pandas-gbq/tests/system/test_to_gbq.py +++ b/packages/pandas-gbq/tests/system/test_to_gbq.py @@ -5,7 +5,6 @@ import datetime import decimal import collections -import functools import random import db_dtypes @@ -23,12 +22,8 @@ def api_method(request): @pytest.fixture -def method_under_test(credentials, project_id): - import pandas_gbq - - return functools.partial( - pandas_gbq.to_gbq, project_id=project_id, credentials=credentials - ) +def method_under_test(to_gbq): + return to_gbq SeriesRoundTripTestCase = collections.namedtuple( @@ -98,7 +93,7 @@ def method_under_test(credentials, project_id): def test_series_round_trip( method_under_test, random_dataset_id, - bigquery_client, + read_gbq, input_series, api_method, api_methods, @@ -114,7 +109,7 @@ def test_series_round_trip( ) method_under_test(df, table_id, api_method=api_method) - round_trip = bigquery_client.list_rows(table_id).to_dataframe() + round_trip = read_gbq(table_id) round_trip_series = round_trip["test_col"].sort_values().reset_index(drop=True) pandas.testing.assert_series_equal( round_trip_series, input_series, check_exact=True, check_names=False, @@ -244,8 +239,8 @@ def test_series_round_trip( ) def test_dataframe_round_trip_with_table_schema( method_under_test, + read_gbq, random_dataset_id, - bigquery_client, input_df, expected_df, table_schema, @@ -260,8 +255,8 @@ def test_dataframe_round_trip_with_table_schema( method_under_test( input_df, table_id, table_schema=table_schema, api_method=api_method ) - round_trip = bigquery_client.list_rows(table_id).to_dataframe( - dtypes=dict(zip(expected_df.columns, expected_df.dtypes)) + round_trip = read_gbq( + table_id, dtypes=dict(zip(expected_df.columns, expected_df.dtypes)), ) round_trip.sort_values("row_num", inplace=True) pandas.testing.assert_frame_equal(expected_df, round_trip) diff --git a/packages/pandas-gbq/tests/unit/conftest.py b/packages/pandas-gbq/tests/unit/conftest.py index cfa1e8199fca..3f0c5e5363cd 100644 --- a/packages/pandas-gbq/tests/unit/conftest.py +++ b/packages/pandas-gbq/tests/unit/conftest.py @@ -26,18 +26,35 @@ def mock_bigquery_client(monkeypatch): # Constructor returns the mock itself, so this mock can be treated as the # constructor or the instance. mock_client.return_value = mock_client - mock_schema = [google.cloud.bigquery.SchemaField("_f0", "INTEGER")] - # Mock out SELECT 1 query results. + mock_query = mock.create_autospec(google.cloud.bigquery.QueryJob) mock_query.job_id = "some-random-id" mock_query.state = "DONE" mock_rows = mock.create_autospec(google.cloud.bigquery.table.RowIterator) mock_rows.total_rows = 1 - mock_rows.schema = mock_schema + mock_rows.__iter__.return_value = [(1,)] mock_query.result.return_value = mock_rows + mock_client.list_rows.return_value = mock_rows mock_client.query.return_value = mock_query # Mock table creation. monkeypatch.setattr(google.cloud.bigquery, "Client", mock_client) mock_client.reset_mock() + + # Mock out SELECT 1 query results. + def generate_schema(): + query = mock_client.query.call_args[0][0] if mock_client.query.call_args else "" + if query == "SELECT 1 AS int_col": + return [google.cloud.bigquery.SchemaField("int_col", "INTEGER")] + else: + return [google.cloud.bigquery.SchemaField("_f0", "INTEGER")] + + type(mock_rows).schema = mock.PropertyMock(side_effect=generate_schema) + + # Mock out get_table. + def get_table(table_ref_or_id, **kwargs): + return google.cloud.bigquery.Table(table_ref_or_id) + + mock_client.get_table.side_effect = get_table + return mock_client diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index 8784a98b4ae6..df9241bc67ca 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -8,6 +8,7 @@ import datetime from unittest import mock +import google.api_core.exceptions import numpy import pandas from pandas import DataFrame @@ -82,6 +83,25 @@ def test__bqschema_to_nullsafe_dtypes(type_, expected): assert result == {"x": expected} +@pytest.mark.parametrize( + ["query_or_table", "expected"], + [ + ("SELECT 1", True), + ("SELECT\n1", True), + ("SELECT\t1", True), + ("dataset.table", False), + (" dataset.table ", False), + ("\r\ndataset.table\r\n", False), + ("project-id.dataset.table", False), + (" project-id.dataset.table ", False), + ("\r\nproject-id.dataset.table\r\n", False), + ], +) +def test__is_query(query_or_table, expected): + result = gbq._is_query(query_or_table) + assert result == expected + + def test_GbqConnector_get_client_w_old_bq(monkeypatch, mock_bigquery_client): gbq._test_google_api_imports() connector = _make_connector() @@ -292,9 +312,10 @@ def test_read_gbq_with_no_project_id_given_should_fail(monkeypatch): gbq.read_gbq("SELECT 1", dialect="standard") -def test_read_gbq_with_inferred_project_id(monkeypatch): +def test_read_gbq_with_inferred_project_id(mock_bigquery_client): df = gbq.read_gbq("SELECT 1", dialect="standard") assert df is not None + mock_bigquery_client.query.assert_called_once() def test_read_gbq_with_inferred_project_id_from_service_account_credentials( @@ -473,7 +494,7 @@ def test_read_gbq_passes_dtypes(mock_bigquery_client, mock_service_account_crede def test_read_gbq_use_bqstorage_api( mock_bigquery_client, mock_service_account_credentials ): - if not FEATURES.bigquery_has_bqstorage: + if not FEATURES.bigquery_has_bqstorage: # pragma: NO COVER pytest.skip("requires BigQuery Storage API") mock_service_account_credentials.project_id = "service_account_project_id" @@ -505,3 +526,73 @@ def test_read_gbq_calls_tqdm(mock_bigquery_client, mock_service_account_credenti _, to_dataframe_kwargs = mock_list_rows.to_dataframe.call_args assert to_dataframe_kwargs["progress_bar_type"] == "foobar" + + +def test_read_gbq_with_full_table_id( + mock_bigquery_client, mock_service_account_credentials +): + mock_service_account_credentials.project_id = "service_account_project_id" + df = gbq.read_gbq( + "my-project.my_dataset.read_gbq_table", + credentials=mock_service_account_credentials, + project_id="param-project", + ) + assert df is not None + + mock_bigquery_client.query.assert_not_called() + sent_table = mock_bigquery_client.list_rows.call_args[0][0] + assert sent_table.project == "my-project" + assert sent_table.dataset_id == "my_dataset" + assert sent_table.table_id == "read_gbq_table" + + +def test_read_gbq_with_partial_table_id( + mock_bigquery_client, mock_service_account_credentials +): + mock_service_account_credentials.project_id = "service_account_project_id" + df = gbq.read_gbq( + "my_dataset.read_gbq_table", + credentials=mock_service_account_credentials, + project_id="param-project", + ) + assert df is not None + + mock_bigquery_client.query.assert_not_called() + sent_table = mock_bigquery_client.list_rows.call_args[0][0] + assert sent_table.project == "param-project" + assert sent_table.dataset_id == "my_dataset" + assert sent_table.table_id == "read_gbq_table" + + +def test_read_gbq_bypasses_query_with_table_id_and_max_results( + mock_bigquery_client, mock_service_account_credentials +): + mock_service_account_credentials.project_id = "service_account_project_id" + df = gbq.read_gbq( + "my-project.my_dataset.read_gbq_table", + credentials=mock_service_account_credentials, + max_results=11, + ) + assert df is not None + + mock_bigquery_client.query.assert_not_called() + sent_table = mock_bigquery_client.list_rows.call_args[0][0] + assert sent_table.project == "my-project" + assert sent_table.dataset_id == "my_dataset" + assert sent_table.table_id == "read_gbq_table" + sent_max_results = mock_bigquery_client.list_rows.call_args[1]["max_results"] + assert sent_max_results == 11 + + +def test_read_gbq_with_list_rows_error_translates_exception( + mock_bigquery_client, mock_service_account_credentials +): + mock_bigquery_client.list_rows.side_effect = ( + google.api_core.exceptions.NotFound("table not found"), + ) + + with pytest.raises(gbq.GenericGBQException, match="table not found"): + gbq.read_gbq( + "my-project.my_dataset.read_gbq_table", + credentials=mock_service_account_credentials, + ) From b4ae7b65d820039d58d1d0f88f7c2df679389f7d Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 28 Dec 2021 14:04:01 -0600 Subject: [PATCH 253/519] test: improve `to_gbq` logic unit test coverage (#449) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot --- packages/pandas-gbq/.coveragerc | 2 +- packages/pandas-gbq/noxfile.py | 2 +- packages/pandas-gbq/owlbot.py | 2 +- packages/pandas-gbq/pandas_gbq/gbq.py | 41 +++-- packages/pandas-gbq/pandas_gbq/load.py | 7 +- packages/pandas-gbq/pandas_gbq/schema.py | 14 +- packages/pandas-gbq/setup.py | 5 +- .../pandas-gbq/testing/constraints-3.7.txt | 1 + packages/pandas-gbq/tests/system/test_gbq.py | 5 - packages/pandas-gbq/tests/unit/conftest.py | 59 +++---- packages/pandas-gbq/tests/unit/test_auth.py | 25 ++- .../pandas-gbq/tests/unit/test_context.py | 16 ++ .../pandas-gbq/tests/unit/test_features.py | 19 +++ packages/pandas-gbq/tests/unit/test_gbq.py | 161 +++++++++++++++--- packages/pandas-gbq/tests/unit/test_load.py | 113 ++++++++++++ packages/pandas-gbq/tests/unit/test_schema.py | 37 ++++ .../pandas-gbq/tests/unit/test_timestamp.py | 9 + packages/pandas-gbq/tests/unit/test_to_gbq.py | 122 +++++++++++++ 18 files changed, 542 insertions(+), 98 deletions(-) create mode 100644 packages/pandas-gbq/tests/unit/test_to_gbq.py diff --git a/packages/pandas-gbq/.coveragerc b/packages/pandas-gbq/.coveragerc index 88b85d0364aa..0a3b1cea22d9 100644 --- a/packages/pandas-gbq/.coveragerc +++ b/packages/pandas-gbq/.coveragerc @@ -22,7 +22,7 @@ omit = google/cloud/__init__.py [report] -fail_under = 89 +fail_under = 94 show_missing = True exclude_lines = # Re-enable the standard pragma diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 398b4dc26baf..5e41983b1189 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -259,7 +259,7 @@ def cover(session): test runs (not system test runs), and then erases coverage data. """ session.install("coverage", "pytest-cov") - session.run("coverage", "report", "--show-missing", "--fail-under=89") + session.run("coverage", "report", "--show-missing", "--fail-under=94") session.run("coverage", "erase") diff --git a/packages/pandas-gbq/owlbot.py b/packages/pandas-gbq/owlbot.py index 9849f98f4613..62c9f3c4e354 100644 --- a/packages/pandas-gbq/owlbot.py +++ b/packages/pandas-gbq/owlbot.py @@ -33,7 +33,7 @@ templated_files = common.py_library( unit_test_python_versions=["3.7", "3.8", "3.9", "3.10"], system_test_python_versions=["3.7", "3.8", "3.9", "3.10"], - cov_level=89, + cov_level=94, unit_test_extras=extras, system_test_extras=extras, intersphinx_dependencies={ diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 5dcc3fd0f672..0a18cc3a1e48 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -121,7 +121,20 @@ class InvalidSchema(ValueError): table in BigQuery. """ - pass + def __init__( + self, message: str, local_schema: Dict[str, Any], remote_schema: Dict[str, Any] + ): + super().__init__(message) + self._local_schema = local_schema + self._remote_schema = remote_schema + + @property + def local_schema(self) -> Dict[str, Any]: + return self._local_schema + + @property + def remote_schema(self) -> Dict[str, Any]: + return self._remote_schema class NotFoundException(ValueError): @@ -354,19 +367,12 @@ def sizeof_fmt(num, suffix="B"): return fmt % (num, "Y", suffix) def get_client(self): + import google.api_core.client_info import pandas - try: - # This module was added in google-api-core 1.11.0. - # We don't have a hard requirement on that version, so only - # populate the client_info if available. - import google.api_core.client_info - - client_info = google.api_core.client_info.ClientInfo( - user_agent="pandas-{}".format(pandas.__version__) - ) - except ImportError: - client_info = None + client_info = google.api_core.client_info.ClientInfo( + user_agent="pandas-{}".format(pandas.__version__) + ) # In addition to new enough version of google-api-core, a new enough # version of google-cloud-bigquery is required to populate the @@ -1057,7 +1063,7 @@ def to_gbq( DeprecationWarning, stacklevel=2, ) - elif api_method == "load_csv": + else: warnings.warn( "chunksize will be ignored when using api_method='load_csv' in a future version of pandas-gbq", PendingDeprecationWarning, @@ -1122,12 +1128,14 @@ def to_gbq( ) elif if_exists == "replace": connector.delete_and_recreate_table(dataset_id, table_id, table_schema) - elif if_exists == "append": + else: if not pandas_gbq.schema.schema_is_subset(original_schema, table_schema): raise InvalidSchema( "Please verify that the structure and " "data types in the DataFrame match the " - "schema of the destination table." + "schema of the destination table.", + table_schema, + original_schema, ) # Update the local `table_schema` so mode (NULLABLE/REQUIRED) @@ -1283,9 +1291,6 @@ def delete(self, table_id): """ from google.api_core.exceptions import NotFound - if not self.exists(table_id): - raise NotFoundException("Table does not exist") - table_ref = self._table_ref(table_id) try: self.client.delete_table(table_ref) diff --git a/packages/pandas-gbq/pandas_gbq/load.py b/packages/pandas-gbq/pandas_gbq/load.py index 315ad5cdf217..588a67193b8c 100644 --- a/packages/pandas-gbq/pandas_gbq/load.py +++ b/packages/pandas-gbq/pandas_gbq/load.py @@ -185,6 +185,11 @@ def load_csv_from_file( chunksize: Optional[int], schema: Optional[Dict[str, Any]], ): + """Manually encode a DataFrame to CSV and use the buffer in a load job. + + This method is needed for writing with google-cloud-bigquery versions that + don't implment load_table_from_dataframe with the CSV serialization format. + """ if schema is None: schema = pandas_gbq.schema.generate_bq_schema(dataframe) @@ -203,7 +208,7 @@ def load_chunk(chunk, job_config): finally: chunk_buffer.close() - return load_csv(dataframe, chunksize, bq_schema, load_chunk,) + return load_csv(dataframe, chunksize, bq_schema, load_chunk) def load_chunks( diff --git a/packages/pandas-gbq/pandas_gbq/schema.py b/packages/pandas-gbq/pandas_gbq/schema.py index e2f97455ecaf..cfa1c76575e6 100644 --- a/packages/pandas-gbq/pandas_gbq/schema.py +++ b/packages/pandas-gbq/pandas_gbq/schema.py @@ -21,7 +21,19 @@ def to_pandas_gbq(client_schema): """Given a sequence of :class:`google.cloud.bigquery.schema.SchemaField`, return a schema in pandas-gbq API format. """ - remote_fields = [field_remote.to_api_repr() for field_remote in client_schema] + remote_fields = [ + # Filter out default values. google-cloud-bigquery versions before + # 2.31.0 (https://github.com/googleapis/python-bigquery/pull/557) + # include a description key, even if not explicitly set. This has the + # potential to unset the description unintentionally in cases where + # pandas-gbq is updating the schema. + { + key: value + for key, value in field_remote.to_api_repr().items() + if value is not None + } + for field_remote in client_schema + ] for field in remote_fields: field["type"] = field["type"].upper() field["mode"] = field["mode"].upper() diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index 4be5a72288e8..2e596cc6d78a 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -28,8 +28,9 @@ "pandas >=0.24.2", "pyarrow >=3.0.0, <7.0dev", "pydata-google-auth", - "google-auth", - "google-auth-oauthlib", + "google-api-core >=1.14.0", + "google-auth >=1.4.1", + "google-auth-oauthlib >=0.0.1", # 2.4.* has a bug where waiting for the query can hang indefinitely. # https://github.com/pydata/pandas-gbq/issues/343 "google-cloud-bigquery[bqstorage,pandas] >=1.11.1,<4.0.0dev,!=2.4.*", diff --git a/packages/pandas-gbq/testing/constraints-3.7.txt b/packages/pandas-gbq/testing/constraints-3.7.txt index 6c3080dc63c9..2a500f352902 100644 --- a/packages/pandas-gbq/testing/constraints-3.7.txt +++ b/packages/pandas-gbq/testing/constraints-3.7.txt @@ -6,6 +6,7 @@ # e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", # Then this file should have foo==1.14.0 db-dtypes==0.3.1 +google-api-core==1.14.0 google-auth==1.4.1 google-auth-oauthlib==0.0.1 google-cloud-bigquery==1.11.1 diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index 812d2089454b..67735c53170b 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -1522,11 +1522,6 @@ def test_delete_table(gbq_table): assert not gbq_table.exists("test_delete_table") -def test_delete_table_not_found(gbq_table): - with pytest.raises(gbq.NotFoundException): - gbq_table.delete("test_delete_table_not_found") - - def test_create_table_data_dataset_does_not_exist( project, credentials, gbq_dataset, random_dataset_id ): diff --git a/packages/pandas-gbq/tests/unit/conftest.py b/packages/pandas-gbq/tests/unit/conftest.py index 3f0c5e5363cd..50a9ac1aa95b 100644 --- a/packages/pandas-gbq/tests/unit/conftest.py +++ b/packages/pandas-gbq/tests/unit/conftest.py @@ -9,6 +9,36 @@ import pytest +def mock_get_credentials(*args, **kwargs): + import google.auth.credentials + + mock_credentials = mock.create_autospec(google.auth.credentials.Credentials) + return mock_credentials, "default-project" + + +@pytest.fixture +def mock_service_account_credentials(): + import google.oauth2.service_account + + mock_credentials = mock.create_autospec(google.oauth2.service_account.Credentials) + return mock_credentials + + +@pytest.fixture +def mock_compute_engine_credentials(): + import google.auth.compute_engine + + mock_credentials = mock.create_autospec(google.auth.compute_engine.Credentials) + return mock_credentials + + +@pytest.fixture(autouse=True) +def no_auth(monkeypatch): + import pydata_google_auth + + monkeypatch.setattr(pydata_google_auth, "default", mock_get_credentials) + + @pytest.fixture(autouse=True, scope="function") def reset_context(): import pandas_gbq @@ -20,41 +50,12 @@ def reset_context(): @pytest.fixture(autouse=True) def mock_bigquery_client(monkeypatch): import google.cloud.bigquery - import google.cloud.bigquery.table mock_client = mock.create_autospec(google.cloud.bigquery.Client) # Constructor returns the mock itself, so this mock can be treated as the # constructor or the instance. mock_client.return_value = mock_client - - mock_query = mock.create_autospec(google.cloud.bigquery.QueryJob) - mock_query.job_id = "some-random-id" - mock_query.state = "DONE" - mock_rows = mock.create_autospec(google.cloud.bigquery.table.RowIterator) - mock_rows.total_rows = 1 - - mock_rows.__iter__.return_value = [(1,)] - mock_query.result.return_value = mock_rows - mock_client.list_rows.return_value = mock_rows - mock_client.query.return_value = mock_query - # Mock table creation. monkeypatch.setattr(google.cloud.bigquery, "Client", mock_client) mock_client.reset_mock() - # Mock out SELECT 1 query results. - def generate_schema(): - query = mock_client.query.call_args[0][0] if mock_client.query.call_args else "" - if query == "SELECT 1 AS int_col": - return [google.cloud.bigquery.SchemaField("int_col", "INTEGER")] - else: - return [google.cloud.bigquery.SchemaField("_f0", "INTEGER")] - - type(mock_rows).schema = mock.PropertyMock(side_effect=generate_schema) - - # Mock out get_table. - def get_table(table_ref_or_id, **kwargs): - return google.cloud.bigquery.Table(table_ref_or_id) - - mock_client.get_table.side_effect = get_table - return mock_client diff --git a/packages/pandas-gbq/tests/unit/test_auth.py b/packages/pandas-gbq/tests/unit/test_auth.py index c101942ed950..d44c6380b575 100644 --- a/packages/pandas-gbq/tests/unit/test_auth.py +++ b/packages/pandas-gbq/tests/unit/test_auth.py @@ -28,35 +28,32 @@ def test_get_credentials_default_credentials(monkeypatch): import google.auth import google.auth.credentials import google.cloud.bigquery + import pydata_google_auth - def mock_default_credentials(scopes=None, request=None): - return ( - mock.create_autospec(google.auth.credentials.Credentials), - "default-project", - ) + mock_user_credentials = mock.create_autospec(google.auth.credentials.Credentials) + + def mock_default_credentials(scopes, **kwargs): + return (mock_user_credentials, "test-project") - monkeypatch.setattr(google.auth, "default", mock_default_credentials) + monkeypatch.setattr(pydata_google_auth, "default", mock_default_credentials) credentials, project = auth.get_credentials() - assert project == "default-project" + assert project == "test-project" assert credentials is not None def test_get_credentials_load_user_no_default(monkeypatch): import google.auth import google.auth.credentials + import pydata_google_auth import pydata_google_auth.cache - def mock_default_credentials(scopes=None, request=None): - return (None, None) - - monkeypatch.setattr(google.auth, "default", mock_default_credentials) mock_user_credentials = mock.create_autospec(google.auth.credentials.Credentials) - mock_cache = mock.create_autospec(pydata_google_auth.cache.CredentialsCache) - mock_cache.load.return_value = mock_user_credentials + def mock_default_credentials(scopes, **kwargs): + return (mock_user_credentials, None) - monkeypatch.setattr(auth, "get_credentials_cache", lambda _: mock_cache) + monkeypatch.setattr(pydata_google_auth, "default", mock_default_credentials) credentials, project = auth.get_credentials() assert project is None diff --git a/packages/pandas-gbq/tests/unit/test_context.py b/packages/pandas-gbq/tests/unit/test_context.py index c0521745f51c..1cf420f0cbca 100644 --- a/packages/pandas-gbq/tests/unit/test_context.py +++ b/packages/pandas-gbq/tests/unit/test_context.py @@ -6,9 +6,25 @@ from unittest import mock +import google.cloud.bigquery +import google.cloud.bigquery.table import pytest +@pytest.fixture(autouse=True) +def default_bigquery_client(mock_bigquery_client): + mock_query = mock.create_autospec(google.cloud.bigquery.QueryJob) + mock_query.job_id = "some-random-id" + mock_query.state = "DONE" + mock_rows = mock.create_autospec(google.cloud.bigquery.table.RowIterator) + mock_rows.total_rows = 1 + mock_rows.__iter__.return_value = [(1,)] + mock_query.result.return_value = mock_rows + mock_bigquery_client.list_rows.return_value = mock_rows + mock_bigquery_client.query.return_value = mock_query + return mock_bigquery_client + + @pytest.fixture(autouse=True) def mock_get_credentials(monkeypatch): from pandas_gbq import auth diff --git a/packages/pandas-gbq/tests/unit/test_features.py b/packages/pandas-gbq/tests/unit/test_features.py index b10b0fa8afdf..d62480f30479 100644 --- a/packages/pandas-gbq/tests/unit/test_features.py +++ b/packages/pandas-gbq/tests/unit/test_features.py @@ -10,6 +10,7 @@ @pytest.fixture(autouse=True) def fresh_bigquery_version(monkeypatch): monkeypatch.setattr(FEATURES, "_bigquery_installed_version", None) + monkeypatch.setattr(FEATURES, "_pandas_installed_version", None) @pytest.mark.parametrize( @@ -28,3 +29,21 @@ def test_bigquery_has_from_dataframe_with_csv(monkeypatch, bigquery_version, exp monkeypatch.setattr(google.cloud.bigquery, "__version__", bigquery_version) assert FEATURES.bigquery_has_from_dataframe_with_csv == expected + + +@pytest.mark.parametrize( + ["pandas_version", "expected"], + [ + ("0.14.7", False), + ("0.22.1", False), + ("0.23.0", True), + ("0.23.1", True), + ("1.0.0", True), + ("2.1.3", True), + ], +) +def test_pandas_has_deprecated_verbose(monkeypatch, pandas_version, expected): + import pandas + + monkeypatch.setattr(pandas, "__version__", pandas_version) + assert FEATURES.pandas_has_deprecated_verbose == expected diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index df9241bc67ca..8740f618c852 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -32,34 +32,40 @@ def mock_get_credentials_no_project(*args, **kwargs): return mock_credentials, None -def mock_get_credentials(*args, **kwargs): - import google.auth.credentials - - mock_credentials = mock.create_autospec(google.auth.credentials.Credentials) - return mock_credentials, "default-project" - - -@pytest.fixture -def mock_service_account_credentials(): - import google.oauth2.service_account - - mock_credentials = mock.create_autospec(google.oauth2.service_account.Credentials) - return mock_credentials - - -@pytest.fixture -def mock_compute_engine_credentials(): - import google.auth.compute_engine +@pytest.fixture(autouse=True) +def default_bigquery_client(mock_bigquery_client): + mock_query = mock.create_autospec(google.cloud.bigquery.QueryJob) + mock_query.job_id = "some-random-id" + mock_query.state = "DONE" + mock_rows = mock.create_autospec(google.cloud.bigquery.table.RowIterator) + mock_rows.total_rows = 1 + + mock_rows.__iter__.return_value = [(1,)] + mock_query.result.return_value = mock_rows + mock_bigquery_client.list_rows.return_value = mock_rows + mock_bigquery_client.query.return_value = mock_query + + # Mock out SELECT 1 query results. + def generate_schema(): + query = ( + mock_bigquery_client.query.call_args[0][0] + if mock_bigquery_client.query.call_args + else "" + ) + if query == "SELECT 1 AS int_col": + return [google.cloud.bigquery.SchemaField("int_col", "INTEGER")] + else: + return [google.cloud.bigquery.SchemaField("_f0", "INTEGER")] - mock_credentials = mock.create_autospec(google.auth.compute_engine.Credentials) - return mock_credentials + type(mock_rows).schema = mock.PropertyMock(side_effect=generate_schema) + # Mock out get_table. + def get_table(table_ref_or_id, **kwargs): + return google.cloud.bigquery.Table(table_ref_or_id) -@pytest.fixture(autouse=True) -def no_auth(monkeypatch): - import pydata_google_auth + mock_bigquery_client.get_table.side_effect = get_table - monkeypatch.setattr(pydata_google_auth, "default", mock_get_credentials) + return mock_bigquery_client @pytest.mark.parametrize( @@ -290,7 +296,7 @@ def test_to_gbq_w_project_table(mock_bigquery_client): assert table.project == "project_table" -def test_to_gbq_creates_dataset(mock_bigquery_client): +def test_to_gbq_create_dataset(mock_bigquery_client): import google.api_core.exceptions mock_bigquery_client.get_table.side_effect = google.api_core.exceptions.NotFound( @@ -303,6 +309,111 @@ def test_to_gbq_creates_dataset(mock_bigquery_client): mock_bigquery_client.create_dataset.assert_called_with(mock.ANY) +def test_dataset_create_already_exists_translates_exception(mock_bigquery_client): + connector = gbq._Dataset("my-project") + connector.client = mock_bigquery_client + mock_bigquery_client.get_dataset.return_value = object() + with pytest.raises(gbq.DatasetCreationError): + connector.create("already_exists") + + +def test_dataset_exists_false(mock_bigquery_client): + connector = gbq._Dataset("my-project") + connector.client = mock_bigquery_client + mock_bigquery_client.get_dataset.side_effect = google.api_core.exceptions.NotFound( + "nope" + ) + assert not connector.exists("not_exists") + + +def test_dataset_exists_true(mock_bigquery_client): + connector = gbq._Dataset("my-project") + connector.client = mock_bigquery_client + mock_bigquery_client.get_dataset.return_value = object() + assert connector.exists("yes_exists") + + +def test_dataset_exists_translates_exception(mock_bigquery_client): + connector = gbq._Dataset("my-project") + connector.client = mock_bigquery_client + mock_bigquery_client.get_dataset.side_effect = google.api_core.exceptions.InternalServerError( + "something went wrong" + ) + with pytest.raises(gbq.GenericGBQException): + connector.exists("not_gonna_work") + + +def test_table_create_already_exists(mock_bigquery_client): + connector = gbq._Table("my-project", "my_dataset") + connector.client = mock_bigquery_client + mock_bigquery_client.get_table.return_value = object() + with pytest.raises(gbq.TableCreationError): + connector.create( + "already_exists", {"fields": [{"name": "f", "type": "STRING"}]} + ) + + +def test_table_create_translates_exception(mock_bigquery_client): + connector = gbq._Table("my-project", "my_dataset") + connector.client = mock_bigquery_client + mock_bigquery_client.get_table.side_effect = google.api_core.exceptions.NotFound( + "nope" + ) + mock_bigquery_client.create_table.side_effect = google.api_core.exceptions.InternalServerError( + "something went wrong" + ) + with pytest.raises(gbq.GenericGBQException): + connector.create( + "not_gonna_work", {"fields": [{"name": "f", "type": "STRING"}]} + ) + + +def test_table_delete_notfound_ok(mock_bigquery_client): + connector = gbq._Table("my-project", "my_dataset") + connector.client = mock_bigquery_client + mock_bigquery_client.delete_table.side_effect = google.api_core.exceptions.NotFound( + "nope" + ) + connector.delete("not_exists") + mock_bigquery_client.delete_table.assert_called_once() + + +def test_table_delete_translates_exception(mock_bigquery_client): + connector = gbq._Table("my-project", "my_dataset") + connector.client = mock_bigquery_client + mock_bigquery_client.delete_table.side_effect = google.api_core.exceptions.InternalServerError( + "something went wrong" + ) + with pytest.raises(gbq.GenericGBQException): + connector.delete("not_gonna_work") + + +def test_table_exists_false(mock_bigquery_client): + connector = gbq._Table("my-project", "my_dataset") + connector.client = mock_bigquery_client + mock_bigquery_client.get_table.side_effect = google.api_core.exceptions.NotFound( + "nope" + ) + assert not connector.exists("not_exists") + + +def test_table_exists_true(mock_bigquery_client): + connector = gbq._Table("my-project", "my_dataset") + connector.client = mock_bigquery_client + mock_bigquery_client.get_table.return_value = object() + assert connector.exists("yes_exists") + + +def test_table_exists_translates_exception(mock_bigquery_client): + connector = gbq._Table("my-project", "my_dataset") + connector.client = mock_bigquery_client + mock_bigquery_client.get_table.side_effect = google.api_core.exceptions.InternalServerError( + "something went wrong" + ) + with pytest.raises(gbq.GenericGBQException): + connector.exists("not_gonna_work") + + def test_read_gbq_with_no_project_id_given_should_fail(monkeypatch): import pydata_google_auth diff --git a/packages/pandas-gbq/tests/unit/test_load.py b/packages/pandas-gbq/tests/unit/test_load.py index 8e18cfb9e5c7..24f262e6ab10 100644 --- a/packages/pandas-gbq/tests/unit/test_load.py +++ b/packages/pandas-gbq/tests/unit/test_load.py @@ -16,6 +16,7 @@ import pandas.testing import pytest +from pandas_gbq import exceptions from pandas_gbq.features import FEATURES from pandas_gbq import load @@ -95,6 +96,85 @@ def test_encode_chunks_with_chunksize_none(): assert len(chunk.index) == 6 +def test_load_csv_from_dataframe_allows_client_to_generate_schema(mock_bigquery_client): + import google.cloud.bigquery + + df = pandas.DataFrame({"int_col": [1, 2, 3]}) + destination = google.cloud.bigquery.TableReference.from_string( + "my-project.my_dataset.my_table" + ) + + _ = list( + load.load_csv_from_dataframe( + mock_bigquery_client, df, destination, None, None, None + ) + ) + + mock_load = mock_bigquery_client.load_table_from_dataframe + assert mock_load.called + _, kwargs = mock_load.call_args + assert "job_config" in kwargs + assert kwargs["job_config"].schema is None + + +def test_load_csv_from_file_generates_schema(mock_bigquery_client): + import google.cloud.bigquery + + df = pandas.DataFrame( + { + "int_col": [1, 2, 3], + "bool_col": [True, False, True], + "float_col": [0.0, 1.25, -2.75], + "string_col": ["a", "b", "c"], + "datetime_col": pandas.Series( + [ + "2021-12-21 13:28:40.123789", + "2000-01-01 11:10:09", + "2040-10-31 23:59:59.999999", + ], + dtype="datetime64[ns]", + ), + "timestamp_col": pandas.Series( + [ + "2021-12-21 13:28:40.123789", + "2000-01-01 11:10:09", + "2040-10-31 23:59:59.999999", + ], + dtype="datetime64[ns]", + ).dt.tz_localize(datetime.timezone.utc), + } + ) + destination = google.cloud.bigquery.TableReference.from_string( + "my-project.my_dataset.my_table" + ) + + _ = list( + load.load_csv_from_file(mock_bigquery_client, df, destination, None, None, None) + ) + + mock_load = mock_bigquery_client.load_table_from_file + assert mock_load.called + _, kwargs = mock_load.call_args + assert "job_config" in kwargs + sent_schema = kwargs["job_config"].schema + assert len(sent_schema) == len(df.columns) + assert sent_schema[0].name == "int_col" + assert sent_schema[0].field_type == "INTEGER" + assert sent_schema[1].name == "bool_col" + assert sent_schema[1].field_type == "BOOLEAN" + assert sent_schema[2].name == "float_col" + assert sent_schema[2].field_type == "FLOAT" + assert sent_schema[3].name == "string_col" + assert sent_schema[3].field_type == "STRING" + # TODO: Disambiguate TIMESTAMP from DATETIME based on if column is + # localized or at least use field type from table metadata. See: + # https://github.com/googleapis/python-bigquery-pandas/issues/450 + assert sent_schema[4].name == "datetime_col" + assert sent_schema[4].field_type == "TIMESTAMP" + assert sent_schema[5].name == "timestamp_col" + assert sent_schema[5].field_type == "TIMESTAMP" + + @pytest.mark.parametrize( ["bigquery_has_from_dataframe_with_csv", "api_method"], [(True, "load_parquet"), (True, "load_csv"), (False, "load_csv")], @@ -143,6 +223,39 @@ def test_load_chunks_with_invalid_api_method(): load.load_chunks(None, None, None, api_method="not_a_thing") +def test_load_parquet_allows_client_to_generate_schema(mock_bigquery_client): + import google.cloud.bigquery + + df = pandas.DataFrame({"int_col": [1, 2, 3]}) + destination = google.cloud.bigquery.TableReference.from_string( + "my-project.my_dataset.my_table" + ) + + load.load_parquet(mock_bigquery_client, df, destination, None, None) + + mock_load = mock_bigquery_client.load_table_from_dataframe + assert mock_load.called + _, kwargs = mock_load.call_args + assert "job_config" in kwargs + assert kwargs["job_config"].schema is None + + +def test_load_parquet_with_bad_conversion(mock_bigquery_client): + import google.cloud.bigquery + import pyarrow + + mock_bigquery_client.load_table_from_dataframe.side_effect = ( + pyarrow.lib.ArrowInvalid() + ) + df = pandas.DataFrame({"int_col": [1, 2, 3]}) + destination = google.cloud.bigquery.TableReference.from_string( + "my-project.my_dataset.my_table" + ) + + with pytest.raises(exceptions.ConversionError): + load.load_parquet(mock_bigquery_client, df, destination, None, None) + + @pytest.mark.parametrize( ("numeric_type",), ( diff --git a/packages/pandas-gbq/tests/unit/test_schema.py b/packages/pandas-gbq/tests/unit/test_schema.py index 743ddc26111c..d31ac2e980fb 100644 --- a/packages/pandas-gbq/tests/unit/test_schema.py +++ b/packages/pandas-gbq/tests/unit/test_schema.py @@ -3,7 +3,9 @@ # license that can be found in the LICENSE file. import datetime +from typing import Any, Dict, List +import google.cloud.bigquery import pandas import pytest @@ -151,3 +153,38 @@ def test_generate_bq_schema(module_under_test, dataframe, expected_schema): def test_update_schema(module_under_test, schema_old, schema_new, expected_output): output = module_under_test.update_schema(schema_old, schema_new) assert output == expected_output + + +@pytest.mark.parametrize( + ["bq_schema", "expected"], + [ + ([], {"fields": []}), + ( + [google.cloud.bigquery.SchemaField("test_col", "STRING")], + {"fields": [{"name": "test_col", "type": "STRING", "mode": "NULLABLE"}]}, + ), + ( + [google.cloud.bigquery.SchemaField("test_col", "STRING", mode="REQUIRED")], + {"fields": [{"name": "test_col", "type": "STRING", "mode": "REQUIRED"}]}, + ), + ( + [ + google.cloud.bigquery.SchemaField("test1", "STRING"), + google.cloud.bigquery.SchemaField("test2", "INTEGER"), + ], + { + "fields": [ + {"name": "test1", "type": "STRING", "mode": "NULLABLE"}, + {"name": "test2", "type": "INTEGER", "mode": "NULLABLE"}, + ] + }, + ), + ], +) +def test_to_pandas_gbq( + bq_schema: List[google.cloud.bigquery.SchemaField], expected: Dict[str, Any] +): + import pandas_gbq.schema + + result = pandas_gbq.schema.to_pandas_gbq(bq_schema) + assert result == expected diff --git a/packages/pandas-gbq/tests/unit/test_timestamp.py b/packages/pandas-gbq/tests/unit/test_timestamp.py index 406643d07e6b..b35c13074c16 100644 --- a/packages/pandas-gbq/tests/unit/test_timestamp.py +++ b/packages/pandas-gbq/tests/unit/test_timestamp.py @@ -56,6 +56,14 @@ def test_localize_df_with_timestamp_column(module_under_test): dtype="datetime64[ns]", ), "float_col": [0.1, 0.2, 0.3], + "repeated_col": pandas.Series( + [ + ["2011-01-01 01:02:03"], + ["2012-02-02 04:05:06"], + ["2013-03-03 07:08:09"], + ], + dtype="object", + ), } ) expected = df.copy() @@ -64,6 +72,7 @@ def test_localize_df_with_timestamp_column(module_under_test): {"name": "integer_col", "type": "INTEGER"}, {"name": "timestamp_col", "type": "TIMESTAMP"}, {"name": "float_col", "type": "FLOAT"}, + {"name": "repeated_col", "type": "TIMESTAMP", "mode": "REPEATED"}, ] localized = module_under_test.localize_df(df, bq_schema) diff --git a/packages/pandas-gbq/tests/unit/test_to_gbq.py b/packages/pandas-gbq/tests/unit/test_to_gbq.py new file mode 100644 index 000000000000..e488bdb520f3 --- /dev/null +++ b/packages/pandas-gbq/tests/unit/test_to_gbq.py @@ -0,0 +1,122 @@ +# Copyright (c) 2021 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import google.cloud.bigquery +import google.api_core.exceptions +from pandas import DataFrame +import pytest + +from pandas_gbq import gbq +from pandas_gbq.features import FEATURES + + +@pytest.fixture +def expected_load_method(mock_bigquery_client): + if FEATURES.pandas_has_parquet_with_lossless_timestamp: + return mock_bigquery_client.load_table_from_dataframe + return mock_bigquery_client.load_table_from_file + + +def test_to_gbq_create_dataset_with_location(mock_bigquery_client): + mock_bigquery_client.get_table.side_effect = google.api_core.exceptions.NotFound( + "my_table" + ) + mock_bigquery_client.get_dataset.side_effect = google.api_core.exceptions.NotFound( + "my_dataset" + ) + gbq.to_gbq( + DataFrame([[1]]), "my_dataset.my_table", project_id="1234", location="us-west1" + ) + assert mock_bigquery_client.create_dataset.called + args, _ = mock_bigquery_client.create_dataset.call_args + sent_dataset = args[0] + assert sent_dataset.location == "us-west1" + + +def test_to_gbq_create_dataset_translates_exception(mock_bigquery_client): + mock_bigquery_client.get_table.side_effect = google.api_core.exceptions.NotFound( + "my_table" + ) + mock_bigquery_client.get_dataset.side_effect = google.api_core.exceptions.NotFound( + "my_dataset" + ) + mock_bigquery_client.create_dataset.side_effect = google.api_core.exceptions.InternalServerError( + "something went wrong" + ) + + with pytest.raises(gbq.GenericGBQException): + gbq.to_gbq(DataFrame([[1]]), "my_dataset.my_table", project_id="1234") + + +def test_to_gbq_with_if_exists_append(mock_bigquery_client, expected_load_method): + from google.cloud.bigquery import SchemaField + + mock_bigquery_client.get_table.return_value = google.cloud.bigquery.Table( + "myproj.my_dataset.my_table", + schema=( + SchemaField("col_a", "FLOAT", mode="REQUIRED"), + SchemaField("col_b", "STRING", mode="REQUIRED"), + ), + ) + gbq.to_gbq( + DataFrame({"col_a": [0.25, 1.5, -1.0], "col_b": ["a", "b", "c"]}), + "my_dataset.my_table", + project_id="myproj", + if_exists="append", + ) + expected_load_method.assert_called_once() + + +def test_to_gbq_with_if_exists_append_mismatch(mock_bigquery_client): + from google.cloud.bigquery import SchemaField + + mock_bigquery_client.get_table.return_value = google.cloud.bigquery.Table( + "myproj.my_dataset.my_table", + schema=(SchemaField("col_a", "INTEGER"), SchemaField("col_b", "STRING")), + ) + with pytest.raises(gbq.InvalidSchema) as exception_block: + gbq.to_gbq( + DataFrame({"col_a": [0.25, 1.5, -1.0]}), + "my_dataset.my_table", + project_id="myproj", + if_exists="append", + ) + + exc = exception_block.value + assert exc.remote_schema == { + "fields": [ + {"name": "col_a", "type": "INTEGER", "mode": "NULLABLE"}, + {"name": "col_b", "type": "STRING", "mode": "NULLABLE"}, + ] + } + assert exc.local_schema == {"fields": [{"name": "col_a", "type": "FLOAT"}]} + + +def test_to_gbq_with_if_exists_replace(mock_bigquery_client): + mock_bigquery_client.get_table.side_effect = ( + # Initial check + google.cloud.bigquery.Table("myproj.my_dataset.my_table"), + # Recreate check + google.api_core.exceptions.NotFound("my_table"), + ) + gbq.to_gbq( + DataFrame([[1]]), + "my_dataset.my_table", + project_id="myproj", + if_exists="replace", + ) + # TODO: We can avoid these API calls by using write disposition in the load + # job. See: https://github.com/googleapis/python-bigquery-pandas/issues/118 + assert mock_bigquery_client.delete_table.called + assert mock_bigquery_client.create_table.called + + +def test_to_gbq_with_if_exists_unknown(): + with pytest.raises(ValueError): + gbq.to_gbq( + DataFrame([[1]]), + "my_dataset.my_table", + project_id="myproj", + if_exists="unknown", + ) From 921be4212939af7e7ce876ede368a8a9f70f86ab Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 29 Dec 2021 11:16:52 -0500 Subject: [PATCH 254/519] chore: update release_level in repo-metadata.json (#451) * chore: update .repo-metadata.json * Update .repo-metadata.json * remove api_shortname Co-authored-by: Tim Swast --- packages/pandas-gbq/.repo-metadata.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/.repo-metadata.json b/packages/pandas-gbq/.repo-metadata.json index fae1dc2089e5..912be418970a 100644 --- a/packages/pandas-gbq/.repo-metadata.json +++ b/packages/pandas-gbq/.repo-metadata.json @@ -2,9 +2,9 @@ "name": "pandas-gbq", "name_pretty": "Google BigQuery connector for pandas", "product_documentation": "https://cloud.google.com/bigquery", - "client_documentation": "https://pandas-gbq.readthedocs.io/en/latest/", + "client_documentation": "https://googleapis.dev/python/pandas-gbq/latest/", "issue_tracker": "https://github.com/googleapis/python-bigquery-pandas/issues", - "release_level": "beta", + "release_level": "preview", "language": "python", "library_type": "INTEGRATION", "repo": "googleapis/python-bigquery-pandas", From 596b6fa693ae0cdd6a553a82a9dc50c9144f9106 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Wed, 29 Dec 2021 17:50:15 +0100 Subject: [PATCH 255/519] chore(deps): update all dependencies (#419) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [![WhiteSource Renovate](https://app.renovatebot.com/images/banner.svg)](https://renovatebot.com) This PR contains the following updates: | Package | Change | Age | Adoption | Passing | Confidence | |---|---|---|---|---|---| | [google-cloud-bigquery](https://togithub.com/googleapis/python-bigquery) | `==2.30.1` -> `==2.31.0` | [![age](https://badges.renovateapi.com/packages/pypi/google-cloud-bigquery/2.31.0/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/pypi/google-cloud-bigquery/2.31.0/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/pypi/google-cloud-bigquery/2.31.0/compatibility-slim/2.30.1)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/pypi/google-cloud-bigquery/2.31.0/confidence-slim/2.30.1)](https://docs.renovatebot.com/merge-confidence/) | | [google-cloud-bigquery-storage](https://togithub.com/googleapis/python-bigquery-storage) | `==2.10.0` -> `==2.10.1` | [![age](https://badges.renovateapi.com/packages/pypi/google-cloud-bigquery-storage/2.10.1/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/pypi/google-cloud-bigquery-storage/2.10.1/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/pypi/google-cloud-bigquery-storage/2.10.1/compatibility-slim/2.10.0)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/pypi/google-cloud-bigquery-storage/2.10.1/confidence-slim/2.10.0)](https://docs.renovatebot.com/merge-confidence/) | | [google-cloud-testutils](https://togithub.com/googleapis/python-test-utils) | `==1.1.0` -> `==1.3.1` | [![age](https://badges.renovateapi.com/packages/pypi/google-cloud-testutils/1.3.1/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/pypi/google-cloud-testutils/1.3.1/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/pypi/google-cloud-testutils/1.3.1/compatibility-slim/1.1.0)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/pypi/google-cloud-testutils/1.3.1/confidence-slim/1.1.0)](https://docs.renovatebot.com/merge-confidence/) | | [pandas](https://pandas.pydata.org) ([source](https://togithub.com/pandas-dev/pandas)) | `==1.3.4` -> `==1.3.5` | [![age](https://badges.renovateapi.com/packages/pypi/pandas/1.3.5/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/pypi/pandas/1.3.5/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/pypi/pandas/1.3.5/compatibility-slim/1.3.4)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/pypi/pandas/1.3.5/confidence-slim/1.3.4)](https://docs.renovatebot.com/merge-confidence/) | | [pyarrow](https://arrow.apache.org/) | `==6.0.0` -> `==6.0.1` | [![age](https://badges.renovateapi.com/packages/pypi/pyarrow/6.0.1/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/pypi/pyarrow/6.0.1/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/pypi/pyarrow/6.0.1/compatibility-slim/6.0.0)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/pypi/pyarrow/6.0.1/confidence-slim/6.0.0)](https://docs.renovatebot.com/merge-confidence/) | --- ### Release Notes
googleapis/python-bigquery ### [`v2.31.0`](https://togithub.com/googleapis/python-bigquery/blob/HEAD/CHANGELOG.md#​2310-httpswwwgithubcomgoogleapispython-bigquerycomparev2301v2310-2021-11-24) [Compare Source](https://togithub.com/googleapis/python-bigquery/compare/v2.30.1...v2.31.0) ##### Features - allow cell magic body to be a $variable ([#​1053](https://www.togithub.com/googleapis/python-bigquery/issues/1053)) ([3a681e0](https://www.github.com/googleapis/python-bigquery/commit/3a681e046819df18118aa0b2b5733416d004c9b3)) - promote `RowIterator.to_arrow_iterable` to public method ([#​1073](https://www.togithub.com/googleapis/python-bigquery/issues/1073)) ([21cd710](https://www.github.com/googleapis/python-bigquery/commit/21cd71022d60c32104f8f90ee2ca445fbb43f7f3)) ##### Bug Fixes - apply timeout to all resumable upload requests ([#​1070](https://www.togithub.com/googleapis/python-bigquery/issues/1070)) ([3314dfb](https://www.github.com/googleapis/python-bigquery/commit/3314dfbed62488503dc41b11e403a672fcf71048)) ##### Dependencies - support OpenTelemetry >= 1.1.0 ([#​1050](https://www.togithub.com/googleapis/python-bigquery/issues/1050)) ([4616cd5](https://www.github.com/googleapis/python-bigquery/commit/4616cd58d3c6da641fb881ce99a87dcdedc20ba2)) ##### [2.30.1](https://www.github.com/googleapis/python-bigquery/compare/v2.30.0...v2.30.1) (2021-11-04) ##### Bug Fixes - error if eval()-ing repr(SchemaField) ([#​1046](https://www.togithub.com/googleapis/python-bigquery/issues/1046)) ([13ac860](https://www.github.com/googleapis/python-bigquery/commit/13ac860de689ea13b35932c67042bc35e388cb30)) ##### Documentation - show gcloud command to authorize against sheets ([#​1045](https://www.togithub.com/googleapis/python-bigquery/issues/1045)) ([20c9024](https://www.github.com/googleapis/python-bigquery/commit/20c9024b5760f7ae41301f4da54568496922cbe2)) - use stable URL for pandas intersphinx links ([#​1048](https://www.togithub.com/googleapis/python-bigquery/issues/1048)) ([73312f8](https://www.github.com/googleapis/python-bigquery/commit/73312f8f0f22ff9175a4f5f7db9bb438a496c164))
googleapis/python-bigquery-storage ### [`v2.10.1`](https://togithub.com/googleapis/python-bigquery-storage/blob/HEAD/CHANGELOG.md#​2101-httpswwwgithubcomgoogleapispython-bigquery-storagecomparev2100v2101-2021-11-11) [Compare Source](https://togithub.com/googleapis/python-bigquery-storage/compare/v2.10.0...v2.10.1)
googleapis/python-test-utils ### [`v1.3.1`](https://togithub.com/googleapis/python-test-utils/blob/HEAD/CHANGELOG.md#​131-httpswwwgithubcomgoogleapispython-test-utilscomparev130v131-2021-12-07) [Compare Source](https://togithub.com/googleapis/python-test-utils/compare/v1.3.0...v1.3.1) ### [`v1.3.0`](https://togithub.com/googleapis/python-test-utils/blob/HEAD/CHANGELOG.md#​130-httpswwwgithubcomgoogleapispython-test-utilscomparev120v130-2021-11-16) [Compare Source](https://togithub.com/googleapis/python-test-utils/compare/v1.2.0...v1.3.0) ##### Features - add 'py.typed' declaration ([#​73](https://www.togithub.com/googleapis/python-test-utils/issues/73)) ([f8f5f0a](https://www.github.com/googleapis/python-test-utils/commit/f8f5f0a194b2420b2fee1cf88ac50220d3ba1538)) ### [`v1.2.0`](https://togithub.com/googleapis/python-test-utils/blob/HEAD/CHANGELOG.md#​120-httpswwwgithubcomgoogleapispython-test-utilscomparev110v120-2021-10-18) [Compare Source](https://togithub.com/googleapis/python-test-utils/compare/v1.1.0...v1.2.0) ##### Features - add support for python 3.10 ([#​68](https://www.togithub.com/googleapis/python-test-utils/issues/68)) ([d93b6a1](https://www.github.com/googleapis/python-test-utils/commit/d93b6a11e3bfade2b29ab90ed3bc2c384beb01cd))
pandas-dev/pandas ### [`v1.3.5`](https://togithub.com/pandas-dev/pandas/releases/v1.3.5) [Compare Source](https://togithub.com/pandas-dev/pandas/compare/v1.3.4...v1.3.5) This is a patch release in the 1.3.x series and includes some regression fixes. We recommend that all users upgrade to this version. See the [full whatsnew](https://pandas.pydata.org/pandas-docs/version/1.3.5/whatsnew/v1.3.5.html) for a list of all the changes. The release will be available on the defaults and conda-forge channels: conda install pandas Or via PyPI: python3 -m pip install --upgrade pandas Please report any issues with the release on the [pandas issue tracker](https://togithub.com/pandas-dev/pandas/issues).
--- ### Configuration 📅 **Schedule**: At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Renovate will not automatically rebase this PR, because other commits have been found. 👻 **Immortal**: This PR will be recreated if closed unmerged. Get [config help](https://togithub.com/renovatebot/renovate/discussions) if that's undesired. --- - [ ] If you want to rebase/retry this PR, click this checkbox. --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#github/googleapis/python-bigquery-pandas). --- .../pandas-gbq/samples/snippets/requirements-test.txt | 2 +- packages/pandas-gbq/samples/snippets/requirements.txt | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements-test.txt b/packages/pandas-gbq/samples/snippets/requirements-test.txt index 3bb560ddf8b7..48472e0052ae 100644 --- a/packages/pandas-gbq/samples/snippets/requirements-test.txt +++ b/packages/pandas-gbq/samples/snippets/requirements-test.txt @@ -1,2 +1,2 @@ -google-cloud-testutils==1.1.0 +google-cloud-testutils==1.3.1 pytest==6.2.5 diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 9f5ef29d1656..0c124f0f7887 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,4 +1,4 @@ -google-cloud-bigquery-storage==2.10.0 -google-cloud-bigquery==2.30.1 -pandas==1.3.4 -pyarrow==6.0.0 +google-cloud-bigquery-storage==2.10.1 +google-cloud-bigquery==2.31.0 +pandas==1.3.5 +pyarrow==6.0.1 From ee8208e7e1d03d4c9ed1bd02162dbfca614c5e06 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 5 Jan 2022 16:17:39 -0600 Subject: [PATCH 256/519] fix: `read_gbq` supports extreme DATETIME values such as `0001-01-01 00:00:00` (#444) * fix: read out-of-bounds DATETIME values such as `0001-01-01 00:00:00` deps: require google-cloud-bigquery 1.26.1 or later * feat: accepts a table ID, which downloads the table without a query * revert tests for read_gbq fix which isn't yet resolved * Revert "revert tests for read_gbq fix which isn't yet resolved" This reverts commit 2a76982df7cff48e58d8b1ad7eae19477665cb76. * add todo for next steps * add unit test for table ID read_gbq * add helper for is_query * implement read_gbq with table id * fix remaining tests, don't localalize out-of-bounds timestamp columns * Update pandas_gbq/gbq.py * fix 3.7 unit tests * correct coverage * skip coverage for optional test skip * fix docs build * improve test coverage for error case * as of google-cloud-bigquery 1.11.0, get_table before list_rows is unnecessary * refactor tests * add more scalars * add more types * add failing time test * add test for bignumeric * add test for null values * add epoch timestamps to tests * add post-download dtype conversions * add failing test for desired fix * fix the issue with extreme datetimes * fix constraints * fix tests for empty dataframe * fix tests for older google-cloud-bigquery * ignore index on empty dataframe * add db-dtypes to runtime import checks * document dependencies * remove TODO, since done * remove unnecessary special case for empty dataframe Fixes prerelease test run * remove redundant 'deprecated' from comment --- .../ci/requirements-3.7-0.24.2.conda | 3 +- packages/pandas-gbq/pandas_gbq/features.py | 24 + packages/pandas-gbq/pandas_gbq/gbq.py | 84 +-- packages/pandas-gbq/pandas_gbq/timestamp.py | 5 - packages/pandas-gbq/setup.py | 17 +- .../pandas-gbq/testing/constraints-3.7.txt | 6 +- packages/pandas-gbq/tests/system/test_gbq.py | 336 +--------- .../pandas-gbq/tests/system/test_read_gbq.py | 607 ++++++++++++++++++ .../pandas-gbq/tests/system/test_to_gbq.py | 58 +- .../pandas-gbq/tests/unit/test_features.py | 47 ++ packages/pandas-gbq/tests/unit/test_gbq.py | 21 +- 11 files changed, 793 insertions(+), 415 deletions(-) create mode 100644 packages/pandas-gbq/tests/system/test_read_gbq.py diff --git a/packages/pandas-gbq/ci/requirements-3.7-0.24.2.conda b/packages/pandas-gbq/ci/requirements-3.7-0.24.2.conda index e0323d926b17..a99bd59e8763 100644 --- a/packages/pandas-gbq/ci/requirements-3.7-0.24.2.conda +++ b/packages/pandas-gbq/ci/requirements-3.7-0.24.2.conda @@ -4,7 +4,8 @@ db-dtypes==0.3.1 fastavro flake8 numpy==1.16.6 -google-cloud-bigquery==1.11.1 +google-cloud-bigquery==1.27.2 +google-cloud-bigquery-storage==1.1.0 pyarrow==3.0.0 pydata-google-auth pytest diff --git a/packages/pandas-gbq/pandas_gbq/features.py b/packages/pandas-gbq/pandas_gbq/features.py index 4259eaf12198..77535041e873 100644 --- a/packages/pandas-gbq/pandas_gbq/features.py +++ b/packages/pandas-gbq/pandas_gbq/features.py @@ -8,7 +8,10 @@ BIGQUERY_MINIMUM_VERSION = "1.11.1" BIGQUERY_CLIENT_INFO_VERSION = "1.12.0" BIGQUERY_BQSTORAGE_VERSION = "1.24.0" +BIGQUERY_ACCURATE_TIMESTAMP_VERSION = "2.6.0" BIGQUERY_FROM_DATAFRAME_CSV_VERSION = "2.6.0" +BIGQUERY_SUPPORTS_BIGNUMERIC_VERSION = "2.10.0" +BIGQUERY_NO_DATE_AS_OBJECT_VERSION = "3.0.0dev" PANDAS_VERBOSITY_DEPRECATION_VERSION = "0.23.0" PANDAS_BOOLEAN_DTYPE_VERSION = "1.0.0" PANDAS_PARQUET_LOSSLESS_TIMESTAMP_VERSION = "1.1.0" @@ -42,6 +45,13 @@ def bigquery_installed_version(self): return self._bigquery_installed_version + @property + def bigquery_has_accurate_timestamp(self): + import pkg_resources + + min_version = pkg_resources.parse_version(BIGQUERY_ACCURATE_TIMESTAMP_VERSION) + return self.bigquery_installed_version >= min_version + @property def bigquery_has_client_info(self): import pkg_resources @@ -51,6 +61,13 @@ def bigquery_has_client_info(self): ) return self.bigquery_installed_version >= bigquery_client_info_version + @property + def bigquery_has_bignumeric(self): + import pkg_resources + + min_version = pkg_resources.parse_version(BIGQUERY_SUPPORTS_BIGNUMERIC_VERSION) + return self.bigquery_installed_version >= min_version + @property def bigquery_has_bqstorage(self): import pkg_resources @@ -69,6 +86,13 @@ def bigquery_has_from_dataframe_with_csv(self): ) return self.bigquery_installed_version >= bigquery_from_dataframe_version + @property + def bigquery_needs_date_as_object(self): + import pkg_resources + + max_version = pkg_resources.parse_version(BIGQUERY_NO_DATE_AS_OBJECT_VERSION) + return self.bigquery_installed_version < max_version + @property def pandas_installed_version(self): import pandas diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 0a18cc3a1e48..feca5e2acc9f 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -2,13 +2,13 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. +from datetime import datetime import logging import re import time -import warnings -from datetime import datetime import typing -from typing import Any, Dict, Optional, Union +from typing import Any, Dict, Optional, Sequence, Union +import warnings import numpy as np @@ -37,14 +37,13 @@ import pandas_gbq.schema import pandas_gbq.timestamp - -logger = logging.getLogger(__name__) - try: import tqdm # noqa except ImportError: tqdm = None +logger = logging.getLogger(__name__) + def _test_google_api_imports(): try: @@ -52,6 +51,11 @@ def _test_google_api_imports(): except ImportError as ex: raise ImportError("pandas-gbq requires setuptools") from ex + try: + import db_dtypes # noqa + except ImportError as ex: + raise ImportError("pandas-gbq requires db-dtypes") from ex + try: import pydata_google_auth # noqa except ImportError as ex: @@ -546,6 +550,8 @@ def _download_results( to_dataframe_kwargs = {} if FEATURES.bigquery_has_bqstorage: to_dataframe_kwargs["create_bqstorage_client"] = create_bqstorage_client + if FEATURES.bigquery_needs_date_as_object: + to_dataframe_kwargs["date_as_object"] = True try: schema_fields = [field.to_api_repr() for field in rows_iter.schema] @@ -559,11 +565,7 @@ def _download_results( except self.http_error as ex: self.process_http_error(ex) - if df.empty: - df = _cast_empty_df_dtypes(schema_fields, df) - - # Ensure any TIMESTAMP columns are tz-aware. - df = pandas_gbq.timestamp.localize_df(df, schema_fields) + df = _finalize_dtypes(df, schema_fields) logger.debug("Got {} rows.\n".format(rows_iter.total_rows)) return df @@ -617,23 +619,18 @@ def _bqschema_to_nullsafe_dtypes(schema_fields): See: http://pandas.pydata.org/pandas-docs/dev/missing_data.html #missing-data-casting-rules-and-indexing """ + import db_dtypes + # If you update this mapping, also update the table at # `docs/reading.rst`. dtype_map = { - "DATE": "datetime64[ns]", - "DATETIME": "datetime64[ns]", "FLOAT": np.dtype(float), - "GEOMETRY": "object", "INTEGER": "Int64", - "RECORD": "object", - "STRING": "object", - # datetime.time objects cannot be case to datetime64. - # https://github.com/pydata/pandas-gbq/issues/328 - "TIME": "object", - # pandas doesn't support timezone-aware dtype in DataFrame/Series - # constructors. It's more idiomatic to localize after construction. - # https://github.com/pandas-dev/pandas/issues/25843 - "TIMESTAMP": "datetime64[ns]", + "TIME": db_dtypes.TimeDtype(), + # Note: Other types such as 'datetime64[ns]' and db_types.DateDtype() + # are not included because the pandas range does not align with the + # BigQuery range. We need to attempt a conversion to those types and + # fall back to 'object' when there are out-of-range values. } # Amend dtype_map with newer extension types if pandas version allows. @@ -656,28 +653,43 @@ def _bqschema_to_nullsafe_dtypes(schema_fields): return dtypes -def _cast_empty_df_dtypes(schema_fields, df): - """Cast any columns in an empty dataframe to correct type. +def _finalize_dtypes( + df: "pandas.DataFrame", schema_fields: Sequence[Dict[str, Any]] +) -> "pandas.DataFrame": + """ + Attempt to change the dtypes of those columns that don't map exactly. - In an empty dataframe, pandas cannot choose a dtype unless one is - explicitly provided. The _bqschema_to_nullsafe_dtypes() function only - provides dtypes when the dtype safely handles null values. This means - that empty int64 and boolean columns are incorrectly classified as - ``object``. + For example db_dtypes.DateDtype() and datetime64[ns] cannot represent + 0001-01-01, but they can represent dates within a couple hundred years of + 1970. See: + https://github.com/googleapis/python-bigquery-pandas/issues/365 """ - if not df.empty: - raise ValueError("DataFrame must be empty in order to cast non-nullsafe dtypes") + import db_dtypes + import pandas.api.types - dtype_map = {"BOOLEAN": bool, "INTEGER": np.int64} + # If you update this mapping, also update the table at + # `docs/reading.rst`. + dtype_map = { + "DATE": db_dtypes.DateDtype(), + "DATETIME": "datetime64[ns]", + "TIMESTAMP": "datetime64[ns]", + } for field in schema_fields: - column = str(field["name"]) + # This method doesn't modify ARRAY/REPEATED columns. if field["mode"].upper() == "REPEATED": continue + name = str(field["name"]) dtype = dtype_map.get(field["type"].upper()) - if dtype: - df[column] = df[column].astype(dtype) + + # Avoid deprecated conversion to timezone-naive dtype by only casting + # object dtypes. + if dtype and pandas.api.types.is_object_dtype(df[name]): + df[name] = df[name].astype(dtype, errors="ignore") + + # Ensure any TIMESTAMP columns are tz-aware. + df = pandas_gbq.timestamp.localize_df(df, schema_fields) return df diff --git a/packages/pandas-gbq/pandas_gbq/timestamp.py b/packages/pandas-gbq/pandas_gbq/timestamp.py index c6bb6d93286b..66374881dae6 100644 --- a/packages/pandas-gbq/pandas_gbq/timestamp.py +++ b/packages/pandas-gbq/pandas_gbq/timestamp.py @@ -30,11 +30,6 @@ def localize_df(df, schema_fields): pandas.DataFrame DataFrame with localized TIMESTAMP columns. """ - if len(df.index) == 0: - # If there are no rows, there is nothing to do. - # Fix for https://github.com/pydata/pandas-gbq/issues/299 - return df - for field in schema_fields: column = str(field["name"]) if "mode" in field and field["mode"].upper() == "REPEATED": diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index 2e596cc6d78a..ccee726523fd 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -28,12 +28,19 @@ "pandas >=0.24.2", "pyarrow >=3.0.0, <7.0dev", "pydata-google-auth", - "google-api-core >=1.14.0", - "google-auth >=1.4.1", + # Note: google-api-core and google-auth are also included via transitive + # dependency on google-cloud-bigquery, but this library also uses them + # directly. + "google-api-core >=1.21.0", + "google-auth >=1.18.0", "google-auth-oauthlib >=0.0.1", - # 2.4.* has a bug where waiting for the query can hang indefinitely. - # https://github.com/pydata/pandas-gbq/issues/343 - "google-cloud-bigquery[bqstorage,pandas] >=1.11.1,<4.0.0dev,!=2.4.*", + # Require 1.27.* because it has a fix for out-of-bounds timestamps. See: + # https://github.com/googleapis/python-bigquery/pull/209 and + # https://github.com/googleapis/python-bigquery-pandas/issues/365 + # Exclude 2.4.* because it has a bug where waiting for the query can hang + # indefinitely. https://github.com/pydata/pandas-gbq/issues/343 + "google-cloud-bigquery >=1.27.2,<4.0.0dev,!=2.4.*", + "google-cloud-bigquery-storage >=1.1.0,<3.0.0dev", ] extras = { "tqdm": "tqdm>=4.23.0", diff --git a/packages/pandas-gbq/testing/constraints-3.7.txt b/packages/pandas-gbq/testing/constraints-3.7.txt index 2a500f352902..f0c9a4acd0e4 100644 --- a/packages/pandas-gbq/testing/constraints-3.7.txt +++ b/packages/pandas-gbq/testing/constraints-3.7.txt @@ -6,10 +6,10 @@ # e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", # Then this file should have foo==1.14.0 db-dtypes==0.3.1 -google-api-core==1.14.0 -google-auth==1.4.1 +google-api-core==1.21.0 +google-auth==1.18.0 google-auth-oauthlib==0.0.1 -google-cloud-bigquery==1.11.1 +google-cloud-bigquery==1.27.2 google-cloud-bigquery-storage==1.1.0 numpy==1.16.6 pandas==0.24.2 diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index 67735c53170b..ec588a3e7298 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -11,7 +11,7 @@ import pandas import pandas.api.types import pandas.testing as tm -from pandas import DataFrame, NaT +from pandas import DataFrame try: import pkg_resources # noqa @@ -21,7 +21,6 @@ import pytz from pandas_gbq import gbq -from pandas_gbq.features import FEATURES import pandas_gbq.schema @@ -153,319 +152,6 @@ def setup(self, project, credentials): self.gbq_connector = gbq.GbqConnector(project, credentials=credentials) self.credentials = credentials - def test_should_properly_handle_empty_strings(self, project_id): - query = 'SELECT "" AS empty_string' - df = gbq.read_gbq( - query, - project_id=project_id, - credentials=self.credentials, - dialect="legacy", - ) - tm.assert_frame_equal(df, DataFrame({"empty_string": [""]})) - - def test_should_properly_handle_null_strings(self, project_id): - query = "SELECT STRING(NULL) AS null_string" - df = gbq.read_gbq( - query, - project_id=project_id, - credentials=self.credentials, - dialect="legacy", - ) - tm.assert_frame_equal(df, DataFrame({"null_string": [None]})) - - def test_should_properly_handle_valid_integers(self, project_id): - query = "SELECT CAST(3 AS INT64) AS valid_integer" - df = gbq.read_gbq( - query, - project_id=project_id, - credentials=self.credentials, - dialect="standard", - ) - tm.assert_frame_equal(df, DataFrame({"valid_integer": [3]}, dtype="Int64")) - - def test_should_properly_handle_nullable_integers(self, project_id): - query = """SELECT * FROM - UNNEST([1, NULL]) AS nullable_integer - """ - df = gbq.read_gbq( - query, - project_id=project_id, - credentials=self.credentials, - dialect="standard", - dtypes={"nullable_integer": "Int64"}, - ) - tm.assert_frame_equal( - df, - DataFrame({"nullable_integer": pandas.Series([1, None], dtype="Int64")}), - ) - - def test_should_properly_handle_valid_longs(self, project_id): - query = "SELECT 1 << 62 AS valid_long" - df = gbq.read_gbq( - query, - project_id=project_id, - credentials=self.credentials, - dialect="standard", - ) - tm.assert_frame_equal(df, DataFrame({"valid_long": [1 << 62]}, dtype="Int64")) - - def test_should_properly_handle_nullable_longs(self, project_id): - query = """SELECT * FROM - UNNEST([1 << 62, NULL]) AS nullable_long - """ - df = gbq.read_gbq( - query, - project_id=project_id, - credentials=self.credentials, - dialect="standard", - dtypes={"nullable_long": "Int64"}, - ) - tm.assert_frame_equal( - df, - DataFrame({"nullable_long": pandas.Series([1 << 62, None], dtype="Int64")}), - ) - - def test_should_properly_handle_null_integers(self, project_id): - query = "SELECT CAST(NULL AS INT64) AS null_integer" - df = gbq.read_gbq( - query, - project_id=project_id, - credentials=self.credentials, - dialect="standard", - dtypes={"null_integer": "Int64"}, - ) - tm.assert_frame_equal( - df, DataFrame({"null_integer": pandas.Series([None], dtype="Int64")}), - ) - - def test_should_properly_handle_valid_floats(self, project_id): - from math import pi - - query = "SELECT PI() AS valid_float" - df = gbq.read_gbq( - query, - project_id=project_id, - credentials=self.credentials, - dialect="legacy", - ) - tm.assert_frame_equal(df, DataFrame({"valid_float": [pi]})) - - def test_should_properly_handle_nullable_floats(self, project_id): - from math import pi - - query = """SELECT * FROM - (SELECT PI() AS nullable_float), - (SELECT NULL AS nullable_float)""" - df = gbq.read_gbq( - query, - project_id=project_id, - credentials=self.credentials, - dialect="legacy", - ) - tm.assert_frame_equal(df, DataFrame({"nullable_float": [pi, None]})) - - def test_should_properly_handle_valid_doubles(self, project_id): - from math import pi - - query = "SELECT PI() * POW(10, 307) AS valid_double" - df = gbq.read_gbq( - query, - project_id=project_id, - credentials=self.credentials, - dialect="legacy", - ) - tm.assert_frame_equal(df, DataFrame({"valid_double": [pi * 10 ** 307]})) - - def test_should_properly_handle_nullable_doubles(self, project_id): - from math import pi - - query = """SELECT * FROM - (SELECT PI() * POW(10, 307) AS nullable_double), - (SELECT NULL AS nullable_double)""" - df = gbq.read_gbq( - query, - project_id=project_id, - credentials=self.credentials, - dialect="legacy", - ) - tm.assert_frame_equal( - df, DataFrame({"nullable_double": [pi * 10 ** 307, None]}) - ) - - def test_should_properly_handle_null_floats(self, project_id): - query = """SELECT null_float - FROM UNNEST(ARRAY[NULL, 1.0]) AS null_float - """ - df = gbq.read_gbq( - query, - project_id=project_id, - credentials=self.credentials, - dialect="standard", - ) - tm.assert_frame_equal(df, DataFrame({"null_float": [np.nan, 1.0]})) - - def test_should_properly_handle_date(self, project_id): - query = "SELECT DATE(2003, 1, 4) AS date_col" - df = gbq.read_gbq(query, project_id=project_id, credentials=self.credentials,) - expected = DataFrame( - { - "date_col": pandas.Series( - [datetime.date(2003, 1, 4)], dtype="datetime64[ns]" - ) - }, - ) - tm.assert_frame_equal(df, expected) - - def test_should_properly_handle_time(self, project_id): - query = ( - "SELECT TIME_ADD(TIME(3, 14, 15), INTERVAL 926589 MICROSECOND) AS time_col" - ) - df = gbq.read_gbq(query, project_id=project_id, credentials=self.credentials,) - expected = DataFrame( - { - "time_col": pandas.Series( - [datetime.time(3, 14, 15, 926589)], dtype="object" - ) - }, - ) - tm.assert_frame_equal(df, expected) - - def test_should_properly_handle_timestamp_unix_epoch(self, project_id): - query = 'SELECT TIMESTAMP("1970-01-01 00:00:00") AS unix_epoch' - df = gbq.read_gbq( - query, - project_id=project_id, - credentials=self.credentials, - dialect="legacy", - ) - expected = DataFrame( - {"unix_epoch": ["1970-01-01T00:00:00.000000Z"]}, dtype="datetime64[ns]", - ) - if expected["unix_epoch"].dt.tz is None: - expected["unix_epoch"] = expected["unix_epoch"].dt.tz_localize("UTC") - tm.assert_frame_equal(df, expected) - - def test_should_properly_handle_arbitrary_timestamp(self, project_id): - query = 'SELECT TIMESTAMP("2004-09-15 05:00:00") AS valid_timestamp' - df = gbq.read_gbq( - query, - project_id=project_id, - credentials=self.credentials, - dialect="legacy", - ) - expected = DataFrame( - {"valid_timestamp": ["2004-09-15T05:00:00.000000Z"]}, - dtype="datetime64[ns]", - ) - if expected["valid_timestamp"].dt.tz is None: - expected["valid_timestamp"] = expected["valid_timestamp"].dt.tz_localize( - "UTC" - ) - tm.assert_frame_equal(df, expected) - - def test_should_properly_handle_datetime_unix_epoch(self, project_id): - query = 'SELECT DATETIME("1970-01-01 00:00:00") AS unix_epoch' - df = gbq.read_gbq( - query, - project_id=project_id, - credentials=self.credentials, - dialect="legacy", - ) - tm.assert_frame_equal( - df, - DataFrame({"unix_epoch": ["1970-01-01T00:00:00"]}, dtype="datetime64[ns]"), - ) - - def test_should_properly_handle_arbitrary_datetime(self, project_id): - query = 'SELECT DATETIME("2004-09-15 05:00:00") AS valid_timestamp' - df = gbq.read_gbq( - query, - project_id=project_id, - credentials=self.credentials, - dialect="legacy", - ) - tm.assert_frame_equal( - df, DataFrame({"valid_timestamp": [np.datetime64("2004-09-15T05:00:00")]}), - ) - - @pytest.mark.parametrize( - "expression, is_expected_dtype", - [ - ("current_date()", pandas.api.types.is_datetime64_ns_dtype), - ("current_timestamp()", pandas.api.types.is_datetime64tz_dtype), - ("current_datetime()", pandas.api.types.is_datetime64_ns_dtype), - ("TRUE", pandas.api.types.is_bool_dtype), - ("FALSE", pandas.api.types.is_bool_dtype), - ], - ) - def test_return_correct_types(self, project_id, expression, is_expected_dtype): - """ - All type checks can be added to this function using additional - parameters, rather than creating additional functions. - We can consolidate the existing functions here in time - - TODO: time doesn't currently parse - ("time(12,30,00)", " Date: Thu, 6 Jan 2022 09:37:40 -0600 Subject: [PATCH 257/519] test: improve test coverage to 96%, remove unused FEATURES properties (#456) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test: improve test coverage by 1%, remove unused FEATURES properties * add test session with no extras * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * noextras take 2 * regex fix * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * remove dead branches * no cover for impossible branches * move unit tests to correct directory Co-authored-by: Owl Bot --- packages/pandas-gbq/.coveragerc | 2 +- packages/pandas-gbq/noxfile.py | 8 ++- packages/pandas-gbq/owlbot.py | 12 +++- packages/pandas-gbq/pandas_gbq/features.py | 22 +------ packages/pandas-gbq/pandas_gbq/gbq.py | 64 ++++++------------- packages/pandas-gbq/tests/system/test_gbq.py | 15 ----- .../pandas-gbq/tests/unit/test_features.py | 20 ++++-- packages/pandas-gbq/tests/unit/test_gbq.py | 42 ++++++------ packages/pandas-gbq/tests/unit/test_to_gbq.py | 19 ++++++ 9 files changed, 92 insertions(+), 112 deletions(-) diff --git a/packages/pandas-gbq/.coveragerc b/packages/pandas-gbq/.coveragerc index 0a3b1cea22d9..d6261761e1f4 100644 --- a/packages/pandas-gbq/.coveragerc +++ b/packages/pandas-gbq/.coveragerc @@ -22,7 +22,7 @@ omit = google/cloud/__init__.py [report] -fail_under = 94 +fail_under = 96 show_missing = True exclude_lines = # Re-enable the standard pragma diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 5e41983b1189..1b719448b518 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -95,7 +95,11 @@ def default(session): constraints_path, ) - session.install("-e", ".[tqdm]", "-c", constraints_path) + if session.python == "3.9": + extras = "" + else: + extras = "[tqdm]" + session.install("-e", f".{extras}", "-c", constraints_path) # Run py.test against the unit tests. session.run( @@ -259,7 +263,7 @@ def cover(session): test runs (not system test runs), and then erases coverage data. """ session.install("coverage", "pytest-cov") - session.run("coverage", "report", "--show-missing", "--fail-under=94") + session.run("coverage", "report", "--show-missing", "--fail-under=96") session.run("coverage", "erase") diff --git a/packages/pandas-gbq/owlbot.py b/packages/pandas-gbq/owlbot.py index 62c9f3c4e354..3ec9f49c09cc 100644 --- a/packages/pandas-gbq/owlbot.py +++ b/packages/pandas-gbq/owlbot.py @@ -29,12 +29,17 @@ # Add templated files # ---------------------------------------------------------------------------- +extras_by_python = { + # Use a middle version of Python to test when no extras are installed. + "3.9": [] +} extras = ["tqdm"] templated_files = common.py_library( unit_test_python_versions=["3.7", "3.8", "3.9", "3.10"], system_test_python_versions=["3.7", "3.8", "3.9", "3.10"], - cov_level=94, + cov_level=96, unit_test_extras=extras, + unit_test_extras_by_python=extras_by_python, system_test_extras=extras, intersphinx_dependencies={ "pandas": "https://pandas.pydata.org/pandas-docs/stable/", @@ -71,6 +76,11 @@ ["noxfile.py"], "--cov=google", "--cov=pandas_gbq", ) +# Workaround for https://github.com/googleapis/synthtool/issues/1317 +s.replace( + ["noxfile.py"], r'extras = "\[\]"', 'extras = ""', +) + s.replace( ["noxfile.py"], r"@nox.session\(python=DEFAULT_PYTHON_VERSION\)\s+def cover\(session\):", diff --git a/packages/pandas-gbq/pandas_gbq/features.py b/packages/pandas-gbq/pandas_gbq/features.py index 77535041e873..ad20c6402ce4 100644 --- a/packages/pandas-gbq/pandas_gbq/features.py +++ b/packages/pandas-gbq/pandas_gbq/features.py @@ -5,9 +5,7 @@ """Module for checking dependency versions and supported features.""" # https://github.com/googleapis/python-bigquery/blob/master/CHANGELOG.md -BIGQUERY_MINIMUM_VERSION = "1.11.1" -BIGQUERY_CLIENT_INFO_VERSION = "1.12.0" -BIGQUERY_BQSTORAGE_VERSION = "1.24.0" +BIGQUERY_MINIMUM_VERSION = "1.27.2" BIGQUERY_ACCURATE_TIMESTAMP_VERSION = "2.6.0" BIGQUERY_FROM_DATAFRAME_CSV_VERSION = "2.6.0" BIGQUERY_SUPPORTS_BIGNUMERIC_VERSION = "2.10.0" @@ -52,15 +50,6 @@ def bigquery_has_accurate_timestamp(self): min_version = pkg_resources.parse_version(BIGQUERY_ACCURATE_TIMESTAMP_VERSION) return self.bigquery_installed_version >= min_version - @property - def bigquery_has_client_info(self): - import pkg_resources - - bigquery_client_info_version = pkg_resources.parse_version( - BIGQUERY_CLIENT_INFO_VERSION - ) - return self.bigquery_installed_version >= bigquery_client_info_version - @property def bigquery_has_bignumeric(self): import pkg_resources @@ -68,15 +57,6 @@ def bigquery_has_bignumeric(self): min_version = pkg_resources.parse_version(BIGQUERY_SUPPORTS_BIGNUMERIC_VERSION) return self.bigquery_installed_version >= min_version - @property - def bigquery_has_bqstorage(self): - import pkg_resources - - bigquery_bqstorage_version = pkg_resources.parse_version( - BIGQUERY_BQSTORAGE_VERSION - ) - return self.bigquery_installed_version >= bigquery_bqstorage_version - @property def bigquery_has_from_dataframe_with_csv(self): import pkg_resources diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index feca5e2acc9f..0edac95d155e 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -18,21 +18,10 @@ if typing.TYPE_CHECKING: # pragma: NO COVER import pandas -# Required dependencies, but treat as optional so that _test_google_api_imports -# can provide a better error message. -try: - from google.api_core import exceptions as google_exceptions - from google.cloud import bigquery -except ImportError: # pragma: NO COVER - bigquery = None - google_exceptions = None - from pandas_gbq.exceptions import ( AccessDenied, GenericGBQException, - PerformanceWarning, ) -from pandas_gbq import features from pandas_gbq.features import FEATURES import pandas_gbq.schema import pandas_gbq.timestamp @@ -48,32 +37,32 @@ def _test_google_api_imports(): try: import pkg_resources # noqa - except ImportError as ex: + except ImportError as ex: # pragma: NO COVER raise ImportError("pandas-gbq requires setuptools") from ex try: import db_dtypes # noqa - except ImportError as ex: + except ImportError as ex: # pragma: NO COVER raise ImportError("pandas-gbq requires db-dtypes") from ex try: import pydata_google_auth # noqa - except ImportError as ex: + except ImportError as ex: # pragma: NO COVER raise ImportError("pandas-gbq requires pydata-google-auth") from ex try: from google_auth_oauthlib.flow import InstalledAppFlow # noqa - except ImportError as ex: + except ImportError as ex: # pragma: NO COVER raise ImportError("pandas-gbq requires google-auth-oauthlib") from ex try: import google.auth # noqa - except ImportError as ex: + except ImportError as ex: # pragma: NO COVER raise ImportError("pandas-gbq requires google-auth") from ex try: from google.cloud import bigquery # noqa - except ImportError as ex: + except ImportError as ex: # pragma: NO COVER raise ImportError("pandas-gbq requires google-cloud-bigquery") from ex @@ -372,23 +361,17 @@ def sizeof_fmt(num, suffix="B"): def get_client(self): import google.api_core.client_info + from google.cloud import bigquery import pandas client_info = google.api_core.client_info.ClientInfo( user_agent="pandas-{}".format(pandas.__version__) ) - - # In addition to new enough version of google-api-core, a new enough - # version of google-cloud-bigquery is required to populate the - # client_info. - if FEATURES.bigquery_has_client_info: - return bigquery.Client( - project=self.project_id, - credentials=self.credentials, - client_info=client_info, - ) - - return bigquery.Client(project=self.project_id, credentials=self.credentials) + return bigquery.Client( + project=self.project_id, + credentials=self.credentials, + client_info=client_info, + ) @staticmethod def process_http_error(ex): @@ -404,6 +387,8 @@ def download_table( progress_bar_type: Optional[str] = None, dtypes: Optional[Dict[str, Union[str, Any]]] = None, ) -> "pandas.DataFrame": + from google.cloud import bigquery + self._start_timer() try: @@ -424,6 +409,7 @@ def download_table( def run_query(self, query, max_results=None, progress_bar_type=None, **kwargs): from concurrent.futures import TimeoutError from google.auth.exceptions import RefreshError + from google.cloud import bigquery job_config = { "query": { @@ -529,27 +515,11 @@ def _download_results( if user_dtypes is None: user_dtypes = {} - if self.use_bqstorage_api and not FEATURES.bigquery_has_bqstorage: - warnings.warn( - ( - "use_bqstorage_api was set, but have google-cloud-bigquery " - "version {}. Requires google-cloud-bigquery version " - "{} or later." - ).format( - FEATURES.bigquery_installed_version, - features.BIGQUERY_BQSTORAGE_VERSION, - ), - PerformanceWarning, - stacklevel=4, - ) - create_bqstorage_client = self.use_bqstorage_api if max_results is not None: create_bqstorage_client = False to_dataframe_kwargs = {} - if FEATURES.bigquery_has_bqstorage: - to_dataframe_kwargs["create_bqstorage_client"] = create_bqstorage_client if FEATURES.bigquery_needs_date_as_object: to_dataframe_kwargs["date_as_object"] = True @@ -560,6 +530,7 @@ def _download_results( df = rows_iter.to_dataframe( dtypes=conversion_dtypes, progress_bar_type=progress_bar_type, + create_bqstorage_client=create_bqstorage_client, **to_dataframe_kwargs, ) except self.http_error as ex: @@ -1051,6 +1022,9 @@ def to_gbq( _test_google_api_imports() + from google.api_core import exceptions as google_exceptions + from google.cloud import bigquery + if verbose is not None and FEATURES.pandas_has_deprecated_verbose: warnings.warn( "verbose is deprecated and will be removed in " diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index ec588a3e7298..214b1f7457fa 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -500,21 +500,6 @@ def test_timeout_configuration(self, project_id): configuration=config, ) - def test_query_response_bytes(self): - assert self.gbq_connector.sizeof_fmt(999) == "999.0 B" - assert self.gbq_connector.sizeof_fmt(1024) == "1.0 KB" - assert self.gbq_connector.sizeof_fmt(1099) == "1.1 KB" - assert self.gbq_connector.sizeof_fmt(1044480) == "1020.0 KB" - assert self.gbq_connector.sizeof_fmt(1048576) == "1.0 MB" - assert self.gbq_connector.sizeof_fmt(1048576000) == "1000.0 MB" - assert self.gbq_connector.sizeof_fmt(1073741824) == "1.0 GB" - assert self.gbq_connector.sizeof_fmt(1.099512e12) == "1.0 TB" - assert self.gbq_connector.sizeof_fmt(1.125900e15) == "1.0 PB" - assert self.gbq_connector.sizeof_fmt(1.152922e18) == "1.0 EB" - assert self.gbq_connector.sizeof_fmt(1.180592e21) == "1.0 ZB" - assert self.gbq_connector.sizeof_fmt(1.208926e24) == "1.0 YB" - assert self.gbq_connector.sizeof_fmt(1.208926e28) == "10000.0 YB" - def test_struct(self, project_id): query = """SELECT 1 int_field, STRUCT("a" as letter, 1 as num) struct_field""" diff --git a/packages/pandas-gbq/tests/unit/test_features.py b/packages/pandas-gbq/tests/unit/test_features.py index c810104f6e06..bfe2ea9b7463 100644 --- a/packages/pandas-gbq/tests/unit/test_features.py +++ b/packages/pandas-gbq/tests/unit/test_features.py @@ -16,8 +16,8 @@ def fresh_bigquery_version(monkeypatch): @pytest.mark.parametrize( ["bigquery_version", "expected"], [ - ("1.11.1", False), - ("1.26.0", False), + ("1.27.2", False), + ("1.99.100", False), ("2.5.4", False), ("2.6.0", True), ("2.6.1", True), @@ -34,8 +34,8 @@ def test_bigquery_has_accurate_timestamp(monkeypatch, bigquery_version, expected @pytest.mark.parametrize( ["bigquery_version", "expected"], [ - ("1.11.1", False), - ("1.26.0", False), + ("1.27.2", False), + ("1.99.100", False), ("2.9.999", False), ("2.10.0", True), ("2.12.0", True), @@ -52,8 +52,8 @@ def test_bigquery_has_bignumeric(monkeypatch, bigquery_version, expected): @pytest.mark.parametrize( ["bigquery_version", "expected"], [ - ("1.11.1", False), - ("1.26.0", False), + ("1.27.2", False), + ("1.99.100", False), ("2.5.4", False), ("2.6.0", True), ("2.6.1", True), @@ -69,7 +69,13 @@ def test_bigquery_has_from_dataframe_with_csv(monkeypatch, bigquery_version, exp @pytest.mark.parametrize( ["bigquery_version", "expected"], - [("1.26.0", True), ("2.12.0", True), ("3.0.0", False), ("3.1.0", False)], + [ + ("1.27.2", True), + ("1.99.100", True), + ("2.12.0", True), + ("3.0.0", False), + ("3.1.0", False), + ], ) def test_bigquery_needs_date_as_object(monkeypatch, bigquery_version, expected): import google.cloud.bigquery diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index 9748595f536a..74bec5ed5c5c 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -109,25 +109,8 @@ def test__is_query(query_or_table, expected): assert result == expected -def test_GbqConnector_get_client_w_old_bq(monkeypatch, mock_bigquery_client): - gbq._test_google_api_imports() - connector = _make_connector() - monkeypatch.setattr( - type(FEATURES), - "bigquery_has_client_info", - mock.PropertyMock(return_value=False), - ) - - connector.get_client() - - # No client_info argument. - mock_bigquery_client.assert_called_with(credentials=mock.ANY, project=mock.ANY) - - def test_GbqConnector_get_client_w_new_bq(mock_bigquery_client): gbq._test_google_api_imports() - if not FEATURES.bigquery_has_client_info: - pytest.skip("google-cloud-bigquery missing client_info feature") pytest.importorskip("google.api_core.client_info") connector = _make_connector() @@ -606,9 +589,6 @@ def test_read_gbq_passes_dtypes(mock_bigquery_client, mock_service_account_crede def test_read_gbq_use_bqstorage_api( mock_bigquery_client, mock_service_account_credentials ): - if not FEATURES.bigquery_has_bqstorage: # pragma: NO COVER - pytest.skip("requires BigQuery Storage API") - mock_service_account_credentials.project_id = "service_account_project_id" df = gbq.read_gbq( "SELECT 1 AS int_col", @@ -716,3 +696,25 @@ def test_read_gbq_with_list_rows_error_translates_exception( "my-project.my_dataset.read_gbq_table", credentials=mock_service_account_credentials, ) + + +@pytest.mark.parametrize( + ["size_in_bytes", "formatted_text"], + [ + (999, "999.0 B"), + (1024, "1.0 KB"), + (1099, "1.1 KB"), + (1044480, "1020.0 KB"), + (1048576, "1.0 MB"), + (1048576000, "1000.0 MB"), + (1073741824, "1.0 GB"), + (1.099512e12, "1.0 TB"), + (1.125900e15, "1.0 PB"), + (1.152922e18, "1.0 EB"), + (1.180592e21, "1.0 ZB"), + (1.208926e24, "1.0 YB"), + (1.208926e28, "10000.0 YB"), + ], +) +def test_query_response_bytes(size_in_bytes, formatted_text): + assert gbq.GbqConnector.sizeof_fmt(size_in_bytes) == formatted_text diff --git a/packages/pandas-gbq/tests/unit/test_to_gbq.py b/packages/pandas-gbq/tests/unit/test_to_gbq.py index e488bdb520f3..22c542f1dcd9 100644 --- a/packages/pandas-gbq/tests/unit/test_to_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_to_gbq.py @@ -49,6 +49,25 @@ def test_to_gbq_create_dataset_translates_exception(mock_bigquery_client): gbq.to_gbq(DataFrame([[1]]), "my_dataset.my_table", project_id="1234") +def test_to_gbq_load_method_translates_exception( + mock_bigquery_client, expected_load_method +): + mock_bigquery_client.get_table.side_effect = google.api_core.exceptions.NotFound( + "my_table" + ) + expected_load_method.side_effect = google.api_core.exceptions.InternalServerError( + "error loading data" + ) + + with pytest.raises(gbq.GenericGBQException): + gbq.to_gbq( + DataFrame({"int_cole": [1, 2, 3]}), + "my_dataset.my_table", + project_id="myproj", + ) + expected_load_method.assert_called_once() + + def test_to_gbq_with_if_exists_append(mock_bigquery_client, expected_load_method): from google.cloud.bigquery import SchemaField From ecb9d101854abf25a4dce43a6607ea48e5eb3068 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 6 Jan 2022 16:22:12 +0000 Subject: [PATCH 258/519] chore: use python-samples-reviewers (#457) --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 2 +- packages/pandas-gbq/.github/CODEOWNERS | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 7519fa3a2289..f33299ddbbab 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:0e18b9475fbeb12d9ad4302283171edebb6baf2dfca1bd215ee3b34ed79d95d7 + digest: sha256:899d5d7cc340fa8ef9d8ae1a8cfba362c6898584f779e156f25ee828ba824610 diff --git a/packages/pandas-gbq/.github/CODEOWNERS b/packages/pandas-gbq/.github/CODEOWNERS index f8714a3e787d..193b4363d07e 100644 --- a/packages/pandas-gbq/.github/CODEOWNERS +++ b/packages/pandas-gbq/.github/CODEOWNERS @@ -8,5 +8,5 @@ # @googleapis/yoshi-python @googleapis/api-bigquery are the default owners for changes in this repo * @googleapis/yoshi-python @googleapis/api-bigquery -# @googleapis/python-samples-owners @googleapis/api-bigquery are the default owners for samples changes -/samples/ @googleapis/python-samples-owners @googleapis/api-bigquery +# @googleapis/python-samples-reviewers @googleapis/api-bigquery are the default owners for samples changes +/samples/ @googleapis/python-samples-reviewers @googleapis/api-bigquery From 6c06524e0228c1ca1bf23f944dcddf8242863723 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 6 Jan 2022 13:53:35 -0600 Subject: [PATCH 259/519] docs: update README to reflect repository move (#454) --- packages/pandas-gbq/README.rst | 47 +++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/packages/pandas-gbq/README.rst b/packages/pandas-gbq/README.rst index 93c87f39026d..1b35f26a91eb 100644 --- a/packages/pandas-gbq/README.rst +++ b/packages/pandas-gbq/README.rst @@ -1,10 +1,21 @@ pandas-gbq ========== -|Build Status| |Version Status| |Coverage Status| |Black Formatted| +|preview| |pypi| |versions| -**pandas-gbq** is a package providing an interface to the Google BigQuery API from pandas +**pandas-gbq** is a package providing an interface to the Google BigQuery API from pandas. +- `Library Documentation`_ +- `Product Documentation`_ + +.. |preview| image:: https://img.shields.io/badge/support-preview-orange.svg + :target: https://github.com/googleapis/google-cloud-python/blob/main/README.rst#beta-support +.. |pypi| image:: https://img.shields.io/pypi/v/pandas-gbq.svg + :target: https://pypi.org/project/pandas-gbq/ +.. |versions| image:: https://img.shields.io/pypi/pyversions/pandas-gbq.svg + :target: https://pypi.org/project/pandas-gbq/ +.. _Library Documentation: https://googleapis.dev/python/pandas-gbq/latest/ +.. _Product Documentation: https://cloud.google.com/bigquery/docs/reference/v2/ Installation ------------ @@ -29,19 +40,31 @@ Install latest development version .. code-block:: shell - $ pip install git+https://github.com/pydata/pandas-gbq.git + $ pip install git+https://github.com/googleapis/python-bigquery-pandas.git Usage ----- -See the `pandas-gbq documentation `_ for more details. +Perform a query +~~~~~~~~~~~~~~~ + +.. code:: python + + import pandas_gbq + + result_dataframe = pandas_gbq.read_gbq("SELECT column FROM dataset.table WHERE value = 'something'") + +Upload a dataframe +~~~~~~~~~~~~~~~~~~ + +.. code:: python + + import pandas_gbq + + pandas_gbq.to_gbq(dataframe, "dataset.table") + +More samples +~~~~~~~~~~~~ -.. |Build Status| image:: https://circleci.com/gh/pydata/pandas-gbq/tree/master.svg?style=svg - :target: https://circleci.com/gh/pydata/pandas-gbq/tree/master -.. |Version Status| image:: https://img.shields.io/pypi/v/pandas-gbq.svg - :target: https://pypi.python.org/pypi/pandas-gbq/ -.. |Coverage Status| image:: https://img.shields.io/codecov/c/github/pydata/pandas-gbq.svg - :target: https://codecov.io/gh/pydata/pandas-gbq/ -.. |Black Formatted| image:: https://img.shields.io/badge/code%20style-black-000000.svg - :target: https://github.com/ambv/black \ No newline at end of file +See the `pandas-gbq documentation `_ for more details. From 3c00455808d68935775d967430ed36f3c6cdc832 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 11 Jan 2022 10:21:11 -0500 Subject: [PATCH 260/519] chore(samples): Add check for tests in directory (#460) Source-Link: https://github.com/googleapis/synthtool/commit/52aef91f8d25223d9dbdb4aebd94ba8eea2101f3 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:36a95b8f494e4674dc9eee9af98961293b51b86b3649942aac800ae6c1f796d4 Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 2 +- .../pandas-gbq/samples/snippets/noxfile.py | 70 +++++++++++-------- 2 files changed, 40 insertions(+), 32 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index f33299ddbbab..6b8a73b31465 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:899d5d7cc340fa8ef9d8ae1a8cfba362c6898584f779e156f25ee828ba824610 + digest: sha256:36a95b8f494e4674dc9eee9af98961293b51b86b3649942aac800ae6c1f796d4 diff --git a/packages/pandas-gbq/samples/snippets/noxfile.py b/packages/pandas-gbq/samples/snippets/noxfile.py index 93a9122cc457..3bbef5d54f44 100644 --- a/packages/pandas-gbq/samples/snippets/noxfile.py +++ b/packages/pandas-gbq/samples/snippets/noxfile.py @@ -14,6 +14,7 @@ from __future__ import print_function +import glob import os from pathlib import Path import sys @@ -184,37 +185,44 @@ def blacken(session: nox.sessions.Session) -> None: def _session_tests( session: nox.sessions.Session, post_install: Callable = None ) -> None: - if TEST_CONFIG["pip_version_override"]: - pip_version = TEST_CONFIG["pip_version_override"] - session.install(f"pip=={pip_version}") - """Runs py.test for a particular project.""" - if os.path.exists("requirements.txt"): - if os.path.exists("constraints.txt"): - session.install("-r", "requirements.txt", "-c", "constraints.txt") - else: - session.install("-r", "requirements.txt") - - if os.path.exists("requirements-test.txt"): - if os.path.exists("constraints-test.txt"): - session.install("-r", "requirements-test.txt", "-c", "constraints-test.txt") - else: - session.install("-r", "requirements-test.txt") - - if INSTALL_LIBRARY_FROM_SOURCE: - session.install("-e", _get_repo_root()) - - if post_install: - post_install(session) - - session.run( - "pytest", - *(PYTEST_COMMON_ARGS + session.posargs), - # Pytest will return 5 when no tests are collected. This can happen - # on travis where slow and flaky tests are excluded. - # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html - success_codes=[0, 5], - env=get_pytest_env_vars(), - ) + # check for presence of tests + test_list = glob.glob("*_test.py") + glob.glob("test_*.py") + if len(test_list) == 0: + print("No tests found, skipping directory.") + else: + if TEST_CONFIG["pip_version_override"]: + pip_version = TEST_CONFIG["pip_version_override"] + session.install(f"pip=={pip_version}") + """Runs py.test for a particular project.""" + if os.path.exists("requirements.txt"): + if os.path.exists("constraints.txt"): + session.install("-r", "requirements.txt", "-c", "constraints.txt") + else: + session.install("-r", "requirements.txt") + + if os.path.exists("requirements-test.txt"): + if os.path.exists("constraints-test.txt"): + session.install( + "-r", "requirements-test.txt", "-c", "constraints-test.txt" + ) + else: + session.install("-r", "requirements-test.txt") + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars(), + ) @nox.session(python=ALL_VERSIONS) From 9d90ed75443a5d55b361c73a55f0c453e31a001b Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 14 Jan 2022 17:06:12 +0000 Subject: [PATCH 261/519] chore(python): update release.sh to use keystore (#464) build: switch to release-please for tagging --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 2 +- packages/pandas-gbq/.github/release-please.yml | 1 + packages/pandas-gbq/.github/release-trigger.yml | 1 + packages/pandas-gbq/.kokoro/release.sh | 2 +- packages/pandas-gbq/.kokoro/release/common.cfg | 12 +++++++++++- 5 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 packages/pandas-gbq/.github/release-trigger.yml diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 6b8a73b31465..eecb84c21b27 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:36a95b8f494e4674dc9eee9af98961293b51b86b3649942aac800ae6c1f796d4 + digest: sha256:ae600f36b6bc972b368367b6f83a1d91ec2c82a4a116b383d67d547c56fe6de3 diff --git a/packages/pandas-gbq/.github/release-please.yml b/packages/pandas-gbq/.github/release-please.yml index 4507ad0598a5..466597e5b196 100644 --- a/packages/pandas-gbq/.github/release-please.yml +++ b/packages/pandas-gbq/.github/release-please.yml @@ -1 +1,2 @@ releaseType: python +handleGHRelease: true diff --git a/packages/pandas-gbq/.github/release-trigger.yml b/packages/pandas-gbq/.github/release-trigger.yml new file mode 100644 index 000000000000..d4ca94189e16 --- /dev/null +++ b/packages/pandas-gbq/.github/release-trigger.yml @@ -0,0 +1 @@ +enabled: true diff --git a/packages/pandas-gbq/.kokoro/release.sh b/packages/pandas-gbq/.kokoro/release.sh index 10fe5b6f44f0..36e4e043d924 100755 --- a/packages/pandas-gbq/.kokoro/release.sh +++ b/packages/pandas-gbq/.kokoro/release.sh @@ -26,7 +26,7 @@ python3 -m pip install --upgrade twine wheel setuptools export PYTHONUNBUFFERED=1 # Move into the package, build the distribution and upload. -TWINE_PASSWORD=$(cat "${KOKORO_GFILE_DIR}/secret_manager/google-cloud-pypi-token") +TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google-cloud-pypi-token-keystore-1") cd github/python-bigquery-pandas python3 setup.py sdist bdist_wheel twine upload --username __token__ --password "${TWINE_PASSWORD}" dist/* diff --git a/packages/pandas-gbq/.kokoro/release/common.cfg b/packages/pandas-gbq/.kokoro/release/common.cfg index a67c994006e8..0fd554538846 100644 --- a/packages/pandas-gbq/.kokoro/release/common.cfg +++ b/packages/pandas-gbq/.kokoro/release/common.cfg @@ -23,8 +23,18 @@ env_vars: { value: "github/python-bigquery-pandas/.kokoro/release.sh" } +# Fetch PyPI password +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "google-cloud-pypi-token-keystore-1" + } + } +} + # Tokens needed to report release status back to GitHub env_vars: { key: "SECRET_MANAGER_KEYS" - value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem,google-cloud-pypi-token" + value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem" } From c8f00f6fd52baf9eb677db82a45e2aff2f2dca22 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 18 Jan 2022 16:07:04 -0600 Subject: [PATCH 262/519] fix: use data project for destination in `to_gbq` (#455) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: use data project for destination in `to_gbq` * bump coverage * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * improve test coverage Co-authored-by: Owl Bot Co-authored-by: Anthonios Partheniou --- packages/pandas-gbq/pandas_gbq/gbq.py | 11 +++-- packages/pandas-gbq/pandas_gbq/load.py | 42 ++++++++++++++++--- packages/pandas-gbq/tests/unit/test_to_gbq.py | 40 ++++++++++++++++++ 3 files changed, 85 insertions(+), 8 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 0edac95d155e..1157c37b4869 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -549,6 +549,7 @@ def load_data( schema=None, progress_bar=True, api_method: str = "load_parquet", + billing_project: Optional[str] = None, ): from pandas_gbq import load @@ -563,6 +564,7 @@ def load_data( schema=schema, location=self.location, api_method=api_method, + billing_project=billing_project, ) if progress_bar and tqdm: chunks = tqdm.tqdm(chunks) @@ -575,8 +577,8 @@ def load_data( except self.http_error as ex: self.process_http_error(ex) - def delete_and_recreate_table(self, dataset_id, table_id, table_schema): - table = _Table(self.project_id, dataset_id, credentials=self.credentials) + def delete_and_recreate_table(self, project_id, dataset_id, table_id, table_schema): + table = _Table(project_id, dataset_id, credentials=self.credentials) table.delete(table_id) table.create(table_id, table_schema) @@ -1113,7 +1115,9 @@ def to_gbq( "'append' or 'replace' data." ) elif if_exists == "replace": - connector.delete_and_recreate_table(dataset_id, table_id, table_schema) + connector.delete_and_recreate_table( + project_id_table, dataset_id, table_id, table_schema + ) else: if not pandas_gbq.schema.schema_is_subset(original_schema, table_schema): raise InvalidSchema( @@ -1142,6 +1146,7 @@ def to_gbq( schema=table_schema, progress_bar=progress_bar, api_method=api_method, + billing_project=project_id, ) diff --git a/packages/pandas-gbq/pandas_gbq/load.py b/packages/pandas-gbq/pandas_gbq/load.py index 588a67193b8c..e52952f2501b 100644 --- a/packages/pandas-gbq/pandas_gbq/load.py +++ b/packages/pandas-gbq/pandas_gbq/load.py @@ -114,6 +114,7 @@ def load_parquet( destination_table_ref: bigquery.TableReference, location: Optional[str], schema: Optional[Dict[str, Any]], + billing_project: Optional[str] = None, ): job_config = bigquery.LoadJobConfig() job_config.write_disposition = "WRITE_APPEND" @@ -126,7 +127,11 @@ def load_parquet( try: client.load_table_from_dataframe( - dataframe, destination_table_ref, job_config=job_config, location=location, + dataframe, + destination_table_ref, + job_config=job_config, + location=location, + project=billing_project, ).result() except pyarrow.lib.ArrowInvalid as exc: raise exceptions.ConversionError( @@ -162,6 +167,7 @@ def load_csv_from_dataframe( location: Optional[str], chunksize: Optional[int], schema: Optional[Dict[str, Any]], + billing_project: Optional[str] = None, ): bq_schema = None @@ -171,7 +177,11 @@ def load_csv_from_dataframe( def load_chunk(chunk, job_config): client.load_table_from_dataframe( - chunk, destination_table_ref, job_config=job_config, location=location, + chunk, + destination_table_ref, + job_config=job_config, + location=location, + project=billing_project, ).result() return load_csv(dataframe, chunksize, bq_schema, load_chunk) @@ -184,6 +194,7 @@ def load_csv_from_file( location: Optional[str], chunksize: Optional[int], schema: Optional[Dict[str, Any]], + billing_project: Optional[str] = None, ): """Manually encode a DataFrame to CSV and use the buffer in a load job. @@ -204,6 +215,7 @@ def load_chunk(chunk, job_config): destination_table_ref, job_config=job_config, location=location, + project=billing_project, ).result() finally: chunk_buffer.close() @@ -219,19 +231,39 @@ def load_chunks( schema=None, location=None, api_method="load_parquet", + billing_project: Optional[str] = None, ): if api_method == "load_parquet": - load_parquet(client, dataframe, destination_table_ref, location, schema) + load_parquet( + client, + dataframe, + destination_table_ref, + location, + schema, + billing_project=billing_project, + ) # TODO: yield progress depending on result() with timeout return [0] elif api_method == "load_csv": if FEATURES.bigquery_has_from_dataframe_with_csv: return load_csv_from_dataframe( - client, dataframe, destination_table_ref, location, chunksize, schema + client, + dataframe, + destination_table_ref, + location, + chunksize, + schema, + billing_project=billing_project, ) else: return load_csv_from_file( - client, dataframe, destination_table_ref, location, chunksize, schema + client, + dataframe, + destination_table_ref, + location, + chunksize, + schema, + billing_project=billing_project, ) else: raise ValueError( diff --git a/packages/pandas-gbq/tests/unit/test_to_gbq.py b/packages/pandas-gbq/tests/unit/test_to_gbq.py index 22c542f1dcd9..a2fa800cc728 100644 --- a/packages/pandas-gbq/tests/unit/test_to_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_to_gbq.py @@ -131,6 +131,46 @@ def test_to_gbq_with_if_exists_replace(mock_bigquery_client): assert mock_bigquery_client.create_table.called +def test_to_gbq_with_if_exists_replace_cross_project( + mock_bigquery_client, expected_load_method +): + mock_bigquery_client.get_table.side_effect = ( + # Initial check + google.cloud.bigquery.Table("data-project.my_dataset.my_table"), + # Recreate check + google.api_core.exceptions.NotFound("my_table"), + ) + gbq.to_gbq( + DataFrame([[1]]), + "data-project.my_dataset.my_table", + project_id="billing-project", + if_exists="replace", + ) + # TODO: We can avoid these API calls by using write disposition in the load + # job. See: https://github.com/googleapis/python-bigquery-pandas/issues/118 + assert mock_bigquery_client.delete_table.called + args, _ = mock_bigquery_client.delete_table.call_args + table_delete: google.cloud.bigquery.TableReference = args[0] + assert table_delete.project == "data-project" + assert table_delete.dataset_id == "my_dataset" + assert table_delete.table_id == "my_table" + assert mock_bigquery_client.create_table.called + args, _ = mock_bigquery_client.create_table.call_args + table_create: google.cloud.bigquery.TableReference = args[0] + assert table_create.project == "data-project" + assert table_create.dataset_id == "my_dataset" + assert table_create.table_id == "my_table" + + # Check that billing project and destination table is set correctly. + expected_load_method.assert_called_once() + load_args, load_kwargs = expected_load_method.call_args + table_destination = load_args[1] + assert table_destination.project == "data-project" + assert table_destination.dataset_id == "my_dataset" + assert table_destination.table_id == "my_table" + assert load_kwargs["project"] == "billing-project" + + def test_to_gbq_with_if_exists_unknown(): with pytest.raises(ValueError): gbq.to_gbq( From 5250f48a5904bf62c5ef1d3c249c9eb5e940fc26 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Wed, 19 Jan 2022 08:39:10 -0500 Subject: [PATCH 263/519] chore(python): Noxfile recognizes that tests can live in a folder (#471) Source-Link: https://github.com/googleapis/synthtool/commit/4760d8dce1351d93658cb11d02a1b7ceb23ae5d7 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:f0e4b51deef56bed74d3e2359c583fc104a8d6367da3984fc5c66938db738828 Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 2 +- packages/pandas-gbq/samples/snippets/noxfile.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index eecb84c21b27..52d79c11f3ad 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:ae600f36b6bc972b368367b6f83a1d91ec2c82a4a116b383d67d547c56fe6de3 + digest: sha256:f0e4b51deef56bed74d3e2359c583fc104a8d6367da3984fc5c66938db738828 diff --git a/packages/pandas-gbq/samples/snippets/noxfile.py b/packages/pandas-gbq/samples/snippets/noxfile.py index 3bbef5d54f44..20cdfc620138 100644 --- a/packages/pandas-gbq/samples/snippets/noxfile.py +++ b/packages/pandas-gbq/samples/snippets/noxfile.py @@ -187,6 +187,7 @@ def _session_tests( ) -> None: # check for presence of tests test_list = glob.glob("*_test.py") + glob.glob("test_*.py") + test_list.extend(glob.glob("tests")) if len(test_list) == 0: print("No tests found, skipping directory.") else: From 1a11cc805b591fdfd166fa99e1355826b50597c5 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 19 Jan 2022 09:53:19 -0600 Subject: [PATCH 264/519] fix: avoid iteritems deprecation in pandas prerelease (#469) --- packages/pandas-gbq/pandas_gbq/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/pandas_gbq/schema.py b/packages/pandas-gbq/pandas_gbq/schema.py index cfa1c76575e6..d3357719b574 100644 --- a/packages/pandas-gbq/pandas_gbq/schema.py +++ b/packages/pandas-gbq/pandas_gbq/schema.py @@ -116,7 +116,7 @@ def generate_bq_schema(dataframe, default_type="STRING"): } fields = [] - for column_name, dtype in dataframe.dtypes.iteritems(): + for column_name, dtype in dataframe.dtypes.items(): fields.append( {"name": column_name, "type": type_mapping.get(dtype.kind, default_type)} ) From 8a5066c5465934543125cdebb77ee590e889e4b2 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 19 Jan 2022 17:12:15 +0000 Subject: [PATCH 265/519] chore(main): release 0.17.0 (#472) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit :robot: I have created a release *beep* *boop* --- ## [0.17.0](https://github.com/googleapis/python-bigquery-pandas/compare/v0.16.0...v0.17.0) (2022-01-19) ### ⚠ BREAKING CHANGES * use nullable Int64 and boolean dtypes if available (#445) ### Features * accepts a table ID, which downloads the table without a query ([#443](https://github.com/googleapis/python-bigquery-pandas/issues/443)) ([bf0e863](https://github.com/googleapis/python-bigquery-pandas/commit/bf0e863ff6506eca267b14e59e47417bd60e947f)) * use nullable Int64 and boolean dtypes if available ([#445](https://github.com/googleapis/python-bigquery-pandas/issues/445)) ([89078f8](https://github.com/googleapis/python-bigquery-pandas/commit/89078f89478469aa60a0a8b8e1e0c4a59aa059e0)) ### Bug Fixes * `read_gbq` supports extreme DATETIME values such as `0001-01-01 00:00:00` ([#444](https://github.com/googleapis/python-bigquery-pandas/issues/444)) ([d120f8f](https://github.com/googleapis/python-bigquery-pandas/commit/d120f8fbdf4541a39ce8d87067523d48f21554bf)) * `to_gbq` allows strings for DATE and floats for NUMERIC with `api_method="load_parquet"` ([#423](https://github.com/googleapis/python-bigquery-pandas/issues/423)) ([2180836](https://github.com/googleapis/python-bigquery-pandas/commit/21808367d02b5b7fcf35b3c7520224c819879aec)) * allow extreme DATE values such as `datetime.date(1, 1, 1)` in `load_gbq` ([#442](https://github.com/googleapis/python-bigquery-pandas/issues/442)) ([e13abaf](https://github.com/googleapis/python-bigquery-pandas/commit/e13abaf015cd1ea9da3ad5063680bf89e18f0fac)) * avoid iteritems deprecation in pandas prerelease ([#469](https://github.com/googleapis/python-bigquery-pandas/issues/469)) ([7379cdc](https://github.com/googleapis/python-bigquery-pandas/commit/7379cdcd7eedcbc751a4002bdf90c12e810e6bcd)) * use data project for destination in `to_gbq` ([#455](https://github.com/googleapis/python-bigquery-pandas/issues/455)) ([891a00c](https://github.com/googleapis/python-bigquery-pandas/commit/891a00c8f202aa476ffb22b2fb92c01ffa84889a)) ### Miscellaneous Chores * release 0.17.0 ([#470](https://github.com/googleapis/python-bigquery-pandas/issues/470)) ([29ac8c3](https://github.com/googleapis/python-bigquery-pandas/commit/29ac8c33127457e86d9864a6979d532cd1d3ae5c)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- packages/pandas-gbq/CHANGELOG.md | 26 +++++++++++++++++++++++ packages/pandas-gbq/pandas_gbq/version.py | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index 318b52410ba8..2a2a2c814321 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,31 @@ # Changelog +## [0.17.0](https://github.com/googleapis/python-bigquery-pandas/compare/v0.16.0...v0.17.0) (2022-01-19) + + +### ⚠ BREAKING CHANGES + +* use nullable Int64 and boolean dtypes if available (#445) + +### Features + +* accepts a table ID, which downloads the table without a query ([#443](https://github.com/googleapis/python-bigquery-pandas/issues/443)) ([bf0e863](https://github.com/googleapis/python-bigquery-pandas/commit/bf0e863ff6506eca267b14e59e47417bd60e947f)) +* use nullable Int64 and boolean dtypes if available ([#445](https://github.com/googleapis/python-bigquery-pandas/issues/445)) ([89078f8](https://github.com/googleapis/python-bigquery-pandas/commit/89078f89478469aa60a0a8b8e1e0c4a59aa059e0)) + + +### Bug Fixes + +* `read_gbq` supports extreme DATETIME values such as `0001-01-01 00:00:00` ([#444](https://github.com/googleapis/python-bigquery-pandas/issues/444)) ([d120f8f](https://github.com/googleapis/python-bigquery-pandas/commit/d120f8fbdf4541a39ce8d87067523d48f21554bf)) +* `to_gbq` allows strings for DATE and floats for NUMERIC with `api_method="load_parquet"` ([#423](https://github.com/googleapis/python-bigquery-pandas/issues/423)) ([2180836](https://github.com/googleapis/python-bigquery-pandas/commit/21808367d02b5b7fcf35b3c7520224c819879aec)) +* allow extreme DATE values such as `datetime.date(1, 1, 1)` in `load_gbq` ([#442](https://github.com/googleapis/python-bigquery-pandas/issues/442)) ([e13abaf](https://github.com/googleapis/python-bigquery-pandas/commit/e13abaf015cd1ea9da3ad5063680bf89e18f0fac)) +* avoid iteritems deprecation in pandas prerelease ([#469](https://github.com/googleapis/python-bigquery-pandas/issues/469)) ([7379cdc](https://github.com/googleapis/python-bigquery-pandas/commit/7379cdcd7eedcbc751a4002bdf90c12e810e6bcd)) +* use data project for destination in `to_gbq` ([#455](https://github.com/googleapis/python-bigquery-pandas/issues/455)) ([891a00c](https://github.com/googleapis/python-bigquery-pandas/commit/891a00c8f202aa476ffb22b2fb92c01ffa84889a)) + + +### Miscellaneous Chores + +* release 0.17.0 ([#470](https://github.com/googleapis/python-bigquery-pandas/issues/470)) ([29ac8c3](https://github.com/googleapis/python-bigquery-pandas/commit/29ac8c33127457e86d9864a6979d532cd1d3ae5c)) + ## [0.16.0](https://www.github.com/googleapis/python-bigquery-pandas/compare/v0.16.0...v0.16.0) (2021-11-08) diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py index 057137c2c114..9fc1e4aa2587 100644 --- a/packages/pandas-gbq/pandas_gbq/version.py +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -2,4 +2,4 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -__version__ = "0.16.0" +__version__ = "0.17.0" From 6600401a5ef642784cff3dde07e0d33900e2bfee Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 21 Jan 2022 08:43:42 -0500 Subject: [PATCH 266/519] ci(python): run lint / unit tests / docs as GH actions (#473) * ci(python): run lint / unit tests / docs as GH actions Source-Link: https://github.com/googleapis/synthtool/commit/57be0cdb0b94e1669cee0ca38d790de1dfdbcd44 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:ed1f9983d5a935a89fe8085e8bb97d94e41015252c5b6c9771257cf8624367e6 * add commit to trigger gh actions * work around bug in templates Co-authored-by: Owl Bot Co-authored-by: Anthonios Partheniou --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 16 +++++- .../pandas-gbq/.github/workflows/docs.yml | 38 +++++++++++++ .../pandas-gbq/.github/workflows/lint.yml | 25 ++++++++ .../pandas-gbq/.github/workflows/unittest.yml | 57 +++++++++++++++++++ packages/pandas-gbq/owlbot.py | 3 + 5 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 packages/pandas-gbq/.github/workflows/docs.yml create mode 100644 packages/pandas-gbq/.github/workflows/lint.yml create mode 100644 packages/pandas-gbq/.github/workflows/unittest.yml diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 52d79c11f3ad..b668c04d5d65 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -1,3 +1,17 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:f0e4b51deef56bed74d3e2359c583fc104a8d6367da3984fc5c66938db738828 + digest: sha256:ed1f9983d5a935a89fe8085e8bb97d94e41015252c5b6c9771257cf8624367e6 + diff --git a/packages/pandas-gbq/.github/workflows/docs.yml b/packages/pandas-gbq/.github/workflows/docs.yml new file mode 100644 index 000000000000..f7b8344c4500 --- /dev/null +++ b/packages/pandas-gbq/.github/workflows/docs.yml @@ -0,0 +1,38 @@ +on: + pull_request: + branches: + - main +name: docs +jobs: + docs: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.10" + - name: Install nox + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install nox + - name: Run docs + run: | + nox -s docs + docfx: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.10" + - name: Install nox + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install nox + - name: Run docfx + run: | + nox -s docfx diff --git a/packages/pandas-gbq/.github/workflows/lint.yml b/packages/pandas-gbq/.github/workflows/lint.yml new file mode 100644 index 000000000000..1e8b05c3d7ff --- /dev/null +++ b/packages/pandas-gbq/.github/workflows/lint.yml @@ -0,0 +1,25 @@ +on: + pull_request: + branches: + - main +name: lint +jobs: + lint: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.10" + - name: Install nox + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install nox + - name: Run lint + run: | + nox -s lint + - name: Run lint_setup_py + run: | + nox -s lint_setup_py diff --git a/packages/pandas-gbq/.github/workflows/unittest.yml b/packages/pandas-gbq/.github/workflows/unittest.yml new file mode 100644 index 000000000000..5d5ffc113fc1 --- /dev/null +++ b/packages/pandas-gbq/.github/workflows/unittest.yml @@ -0,0 +1,57 @@ +on: + pull_request: + branches: + - main +name: unittest +jobs: + unit: + runs-on: ubuntu-latest + strategy: + matrix: + python: ['3.7', '3.8', '3.9', '3.10'] + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + - name: Install nox + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install nox + - name: Run unit tests + env: + COVERAGE_FILE: .coverage-${{ matrix.python }} + run: | + nox -s unit-${{ matrix.python }} + - name: Upload coverage results + uses: actions/upload-artifact@v2 + with: + name: coverage-artifacts + path: .coverage-${{ matrix.python }} + + cover: + runs-on: ubuntu-latest + needs: + - unit + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.10" + - name: Install coverage + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install coverage + - name: Download coverage results + uses: actions/download-artifact@v2 + with: + name: coverage-artifacts + path: .coverage-results/ + - name: Report coverage results + run: | + coverage combine .coverage-results/.coverage* + coverage report --show-missing --fail-under=96 diff --git a/packages/pandas-gbq/owlbot.py b/packages/pandas-gbq/owlbot.py index 3ec9f49c09cc..5cf11d2ec940 100644 --- a/packages/pandas-gbq/owlbot.py +++ b/packages/pandas-gbq/owlbot.py @@ -176,6 +176,9 @@ def cover(session):""", [".github/header-checker-lint.yml"], '"Google LLC"', '"pandas-gbq Authors"', ) +# Work around bug in templates https://github.com/googleapis/synthtool/pull/1335 +s.replace(".github/workflows/unittest.yml", "--fail-under=100", "--fail-under=96") + # ---------------------------------------------------------------------------- # Samples templates # ---------------------------------------------------------------------------- From 2428023eeacb2dc29648f0dcfa576447403a669a Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 26 Jan 2022 09:38:28 -0600 Subject: [PATCH 267/519] docs: document additional breaking change in 0.17.0 (#477) --- packages/pandas-gbq/CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index 2a2a2c814321..c7a1eb16d2cb 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -5,7 +5,8 @@ ### ⚠ BREAKING CHANGES -* use nullable Int64 and boolean dtypes if available (#445) +* the first argument of `read_gbq` is renamed from `query` to `query_or_table` ([#443](https://github.com/googleapis/python-bigquery-pandas/issues/443)) ([bf0e863](https://github.com/googleapis/python-bigquery-pandas/commit/bf0e863ff6506eca267b14e59e47417bd60e947f)) +* use nullable Int64 and boolean dtypes if available ([#445](https://github.com/googleapis/python-bigquery-pandas/issues/445)) ([89078f8](https://github.com/googleapis/python-bigquery-pandas/commit/89078f89478469aa60a0a8b8e1e0c4a59aa059e0)) ### Features From 84a532ff1253a62a55752e1f930f7a804cf5d3b4 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 23 Feb 2022 10:20:34 -0600 Subject: [PATCH 268/519] chore: add custom sync repo settings (#484) No Python 3.6 support in this connector. --- .../.github/sync-repo-settings.yaml | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 packages/pandas-gbq/.github/sync-repo-settings.yaml diff --git a/packages/pandas-gbq/.github/sync-repo-settings.yaml b/packages/pandas-gbq/.github/sync-repo-settings.yaml new file mode 100644 index 000000000000..e62a467931a8 --- /dev/null +++ b/packages/pandas-gbq/.github/sync-repo-settings.yaml @@ -0,0 +1,31 @@ +# https://github.com/googleapis/repo-automation-bots/tree/main/packages/sync-repo-settings +# Rules for main branch protection +branchProtectionRules: +# Identifies the protection rule pattern. Name of the branch to be protected. +# Defaults to `main` +- pattern: main + requiresCodeOwnerReviews: true + requiresStrictStatusChecks: true + requiredStatusCheckContexts: + - 'cla/google' + - 'OwlBot Post Processor' + - 'docs' + - 'lint' + - 'unit (3.7)' + - 'unit (3.8)' + - 'unit (3.9)' + - 'unit (3.10)' + - 'cover' +permissionRules: + - team: actools-python + permission: admin + - team: actools + permission: admin + - team: api-bigquery + permission: push + - team: yoshi-python + permission: push + - team: python-samples-owners + permission: push + - team: python-samples-reviewers + permission: push From d196d5067c524105e5df6ceca3719a4fe462ebae Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 23 Feb 2022 13:23:00 -0600 Subject: [PATCH 269/519] chore: add Kokoro back to required checks (#485) * chore: add Kokoro back to required checks * add samples checks too --- packages/pandas-gbq/.github/sync-repo-settings.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/pandas-gbq/.github/sync-repo-settings.yaml b/packages/pandas-gbq/.github/sync-repo-settings.yaml index e62a467931a8..590dda51fa37 100644 --- a/packages/pandas-gbq/.github/sync-repo-settings.yaml +++ b/packages/pandas-gbq/.github/sync-repo-settings.yaml @@ -16,6 +16,12 @@ branchProtectionRules: - 'unit (3.9)' - 'unit (3.10)' - 'cover' + - 'Kokoro' + - 'Samples - Lint' + - 'Samples - Python 3.7' + - 'Samples - Python 3.8' + - 'Samples - Python 3.9' + - 'Samples - Python 3.10' permissionRules: - team: actools-python permission: admin From 6f1c04f1e4989b50cf41efa02a776c8e5b5a38c5 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 24 Feb 2022 09:26:20 -0600 Subject: [PATCH 270/519] fix: avoid `TypeError` when executing DML statements with `read_gbq` (#483) --- packages/pandas-gbq/pandas_gbq/gbq.py | 6 ++++ .../pandas-gbq/tests/system/test_read_gbq.py | 30 +++++++++++++++++++ packages/pandas-gbq/tests/unit/test_gbq.py | 25 +++++++++++++--- 3 files changed, 57 insertions(+), 4 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 1157c37b4869..6c9b6804f080 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -410,6 +410,7 @@ def run_query(self, query, max_results=None, progress_bar_type=None, **kwargs): from concurrent.futures import TimeoutError from google.auth.exceptions import RefreshError from google.cloud import bigquery + import pandas job_config = { "query": { @@ -495,6 +496,11 @@ def run_query(self, query, max_results=None, progress_bar_type=None, **kwargs): except self.http_error as ex: self.process_http_error(ex) + # Avoid attempting to download results from DML queries, which have no + # destination. + if query_reply.destination is None: + return pandas.DataFrame() + rows_iter = self.client.list_rows( query_reply.destination, max_results=max_results ) diff --git a/packages/pandas-gbq/tests/system/test_read_gbq.py b/packages/pandas-gbq/tests/system/test_read_gbq.py index a13e830f444f..65a65ff73e0a 100644 --- a/packages/pandas-gbq/tests/system/test_read_gbq.py +++ b/packages/pandas-gbq/tests/system/test_read_gbq.py @@ -5,8 +5,10 @@ import collections import datetime import decimal +import random import db_dtypes +from google.cloud import bigquery import pandas import pandas.testing import pytest @@ -21,6 +23,21 @@ ) +@pytest.fixture +def writable_table( + bigquery_client: bigquery.Client, project_id: str, random_dataset: bigquery.Dataset +): + full_table_id = f"{project_id}.{random_dataset.dataset_id}.writable_table_{random.randrange(1_000_000_000)}" + table = bigquery.Table(full_table_id) + table.schema = [ + bigquery.SchemaField("field1", "STRING"), + bigquery.SchemaField("field2", "INTEGER"), + ] + bigquery_client.create_table(table) + yield full_table_id + bigquery_client.delete_table(full_table_id) + + @pytest.mark.parametrize(["use_bqstorage_api"], [(True,), (False,)]) @pytest.mark.parametrize( ["query", "expected", "use_bqstorage_apis"], @@ -605,3 +622,16 @@ def test_empty_dataframe(read_gbq, use_bqstorage_api): ) result = read_gbq(query, use_bqstorage_api=use_bqstorage_api) pandas.testing.assert_frame_equal(result, expected, check_index_type=False) + + +def test_dml_query(read_gbq, writable_table: str): + query = f""" + UPDATE `{writable_table}` + SET field1 = NULL + WHERE field1 = 'string'; + UPDATE `{writable_table}` + SET field2 = NULL + WHERE field2 < 0; + """ + result = read_gbq(query) + assert result is not None diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index 74bec5ed5c5c..511e68d6404b 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -32,18 +32,23 @@ def mock_get_credentials_no_project(*args, **kwargs): return mock_credentials, None -@pytest.fixture(autouse=True) -def default_bigquery_client(mock_bigquery_client): +@pytest.fixture +def mock_query_job(): mock_query = mock.create_autospec(google.cloud.bigquery.QueryJob) mock_query.job_id = "some-random-id" mock_query.state = "DONE" + return mock_query + + +@pytest.fixture(autouse=True) +def default_bigquery_client(mock_bigquery_client, mock_query_job): mock_rows = mock.create_autospec(google.cloud.bigquery.table.RowIterator) mock_rows.total_rows = 1 mock_rows.__iter__.return_value = [(1,)] - mock_query.result.return_value = mock_rows + mock_query_job.result.return_value = mock_rows mock_bigquery_client.list_rows.return_value = mock_rows - mock_bigquery_client.query.return_value = mock_query + mock_bigquery_client.query.return_value = mock_query_job # Mock out SELECT 1 query results. def generate_schema(): @@ -718,3 +723,15 @@ def test_read_gbq_with_list_rows_error_translates_exception( ) def test_query_response_bytes(size_in_bytes, formatted_text): assert gbq.GbqConnector.sizeof_fmt(size_in_bytes) == formatted_text + + +def test_run_query_with_dml_query(mock_bigquery_client, mock_query_job): + """ + Don't attempt to download results from a DML query / query with no results. + + https://github.com/googleapis/python-bigquery-pandas/issues/481 + """ + connector = _make_connector() + type(mock_query_job).destination = mock.PropertyMock(return_value=None) + connector.run_query("UPDATE tablename SET value = '';") + mock_bigquery_client.list_rows.assert_not_called() From 79a06ba2c0f585d2be57e0fc05f362c78fbd38e8 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Thu, 24 Feb 2022 10:45:33 -0600 Subject: [PATCH 271/519] chore(main): release 0.17.1 (#480) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- packages/pandas-gbq/CHANGELOG.md | 12 ++++++++++++ packages/pandas-gbq/pandas_gbq/version.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index c7a1eb16d2cb..20f508762338 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +### [0.17.1](https://github.com/googleapis/python-bigquery-pandas/compare/v0.17.0...v0.17.1) (2022-02-24) + + +### Bug Fixes + +* avoid `TypeError` when executing DML statements with `read_gbq` ([#483](https://github.com/googleapis/python-bigquery-pandas/issues/483)) ([e9f0e3f](https://github.com/googleapis/python-bigquery-pandas/commit/e9f0e3f73f597530b8cf87324b9c4b0b54a79812)) + + +### Documentation + +* document additional breaking change in 0.17.0 ([#477](https://github.com/googleapis/python-bigquery-pandas/issues/477)) ([a858c80](https://github.com/googleapis/python-bigquery-pandas/commit/a858c80a37dc94707a41a6d865af2bc91543328a)) + ## [0.17.0](https://github.com/googleapis/python-bigquery-pandas/compare/v0.16.0...v0.17.0) (2022-01-19) diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py index 9fc1e4aa2587..e0370797b1f0 100644 --- a/packages/pandas-gbq/pandas_gbq/version.py +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -2,4 +2,4 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -__version__ = "0.17.0" +__version__ = "0.17.1" From 629d3a9bce551db8347ad1b84ffcf9d6652e0f1a Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 1 Mar 2022 12:16:21 +0000 Subject: [PATCH 272/519] chore(deps): update actions/setup-python action to v3 (#486) Source-Link: https://github.com/googleapis/synthtool/commit/571ee2c3b26182429eddcf115122ee545d7d3787 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:660abdf857d3ab9aabcd967c163c70e657fcc5653595c709263af5f3fa23ef67 --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 3 +-- packages/pandas-gbq/.github/workflows/docs.yml | 4 ++-- packages/pandas-gbq/.github/workflows/lint.yml | 2 +- packages/pandas-gbq/.github/workflows/unittest.yml | 4 ++-- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index b668c04d5d65..d9a55fa405e8 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:ed1f9983d5a935a89fe8085e8bb97d94e41015252c5b6c9771257cf8624367e6 - + digest: sha256:660abdf857d3ab9aabcd967c163c70e657fcc5653595c709263af5f3fa23ef67 diff --git a/packages/pandas-gbq/.github/workflows/docs.yml b/packages/pandas-gbq/.github/workflows/docs.yml index f7b8344c4500..cca4e98bf236 100644 --- a/packages/pandas-gbq/.github/workflows/docs.yml +++ b/packages/pandas-gbq/.github/workflows/docs.yml @@ -10,7 +10,7 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: "3.10" - name: Install nox @@ -26,7 +26,7 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: "3.10" - name: Install nox diff --git a/packages/pandas-gbq/.github/workflows/lint.yml b/packages/pandas-gbq/.github/workflows/lint.yml index 1e8b05c3d7ff..f687324ef2eb 100644 --- a/packages/pandas-gbq/.github/workflows/lint.yml +++ b/packages/pandas-gbq/.github/workflows/lint.yml @@ -10,7 +10,7 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: "3.10" - name: Install nox diff --git a/packages/pandas-gbq/.github/workflows/unittest.yml b/packages/pandas-gbq/.github/workflows/unittest.yml index 5d5ffc113fc1..d90dbe9c231d 100644 --- a/packages/pandas-gbq/.github/workflows/unittest.yml +++ b/packages/pandas-gbq/.github/workflows/unittest.yml @@ -13,7 +13,7 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python }} - name: Install nox @@ -39,7 +39,7 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: "3.10" - name: Install coverage From 3abd31c4ebb3731277b6e7953b63d7458ca84741 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Tue, 1 Mar 2022 20:55:26 +0100 Subject: [PATCH 273/519] chore(deps): update all dependencies (#463) Co-authored-by: Tim Swast --- packages/pandas-gbq/samples/snippets/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 0c124f0f7887..720188cfe1af 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,4 +1,4 @@ -google-cloud-bigquery-storage==2.10.1 -google-cloud-bigquery==2.31.0 +google-cloud-bigquery-storage==2.11.0 +google-cloud-bigquery==2.32.0 pandas==1.3.5 pyarrow==6.0.1 From c33d8a8fd765110700f1ab5c9ba38161b8d2bdba Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Wed, 2 Mar 2022 12:14:40 -0600 Subject: [PATCH 274/519] chore(deps): update actions/checkout action to v3 (#489) Source-Link: https://github.com/googleapis/synthtool/commit/ca879097772aeec2cbb971c3cea8ecc81522b68a Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:6162c384d685c5fe22521d3f37f6fc732bf99a085f6d47b677dbcae97fc21392 Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 2 +- packages/pandas-gbq/.github/workflows/docs.yml | 4 ++-- packages/pandas-gbq/.github/workflows/lint.yml | 2 +- packages/pandas-gbq/.github/workflows/unittest.yml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index d9a55fa405e8..480226ac08a9 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:660abdf857d3ab9aabcd967c163c70e657fcc5653595c709263af5f3fa23ef67 + digest: sha256:6162c384d685c5fe22521d3f37f6fc732bf99a085f6d47b677dbcae97fc21392 diff --git a/packages/pandas-gbq/.github/workflows/docs.yml b/packages/pandas-gbq/.github/workflows/docs.yml index cca4e98bf236..b46d7305d8cf 100644 --- a/packages/pandas-gbq/.github/workflows/docs.yml +++ b/packages/pandas-gbq/.github/workflows/docs.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Python uses: actions/setup-python@v3 with: @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Python uses: actions/setup-python@v3 with: diff --git a/packages/pandas-gbq/.github/workflows/lint.yml b/packages/pandas-gbq/.github/workflows/lint.yml index f687324ef2eb..f512a4960beb 100644 --- a/packages/pandas-gbq/.github/workflows/lint.yml +++ b/packages/pandas-gbq/.github/workflows/lint.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Python uses: actions/setup-python@v3 with: diff --git a/packages/pandas-gbq/.github/workflows/unittest.yml b/packages/pandas-gbq/.github/workflows/unittest.yml index d90dbe9c231d..0a86d57beb2a 100644 --- a/packages/pandas-gbq/.github/workflows/unittest.yml +++ b/packages/pandas-gbq/.github/workflows/unittest.yml @@ -11,7 +11,7 @@ jobs: python: ['3.7', '3.8', '3.9', '3.10'] steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Python uses: actions/setup-python@v3 with: @@ -37,7 +37,7 @@ jobs: - unit steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Python uses: actions/setup-python@v3 with: From a8e1c88a4b4d9e8776774a8ea094d33a3d61d4e7 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Wed, 2 Mar 2022 20:51:07 +0100 Subject: [PATCH 275/519] deps: allow pyarrow 7.0 (#487) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update all dependencies * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * Update setup.py * limit pandas to 3.8 Co-authored-by: Owl Bot Co-authored-by: Tim Swast --- .../pandas-gbq/samples/snippets/requirements-test.txt | 2 +- packages/pandas-gbq/samples/snippets/requirements.txt | 9 +++++---- packages/pandas-gbq/setup.py | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements-test.txt b/packages/pandas-gbq/samples/snippets/requirements-test.txt index 48472e0052ae..048b49808a9d 100644 --- a/packages/pandas-gbq/samples/snippets/requirements-test.txt +++ b/packages/pandas-gbq/samples/snippets/requirements-test.txt @@ -1,2 +1,2 @@ google-cloud-testutils==1.3.1 -pytest==6.2.5 +pytest==7.0.1 diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 720188cfe1af..254e656f444b 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,4 +1,5 @@ -google-cloud-bigquery-storage==2.11.0 -google-cloud-bigquery==2.32.0 -pandas==1.3.5 -pyarrow==6.0.1 +google-cloud-bigquery-storage==2.12.0 +google-cloud-bigquery==2.34.0 +pandas==1.3.5; python_version < '3.8' +pandas==1.4.1; python_version >= '3.8' +pyarrow==7.0.0 diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index ccee726523fd..fd62b7200ae3 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -26,7 +26,7 @@ "db-dtypes >=0.3.1,<2.0.0", "numpy >=1.16.6", "pandas >=0.24.2", - "pyarrow >=3.0.0, <7.0dev", + "pyarrow >=3.0.0, <8.0dev", "pydata-google-auth", # Note: google-api-core and google-auth are also included via transitive # dependency on google-cloud-bigquery, but this library also uses them From 1c1c43116c8728576c44bbd64644ee3deda3a289 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Thu, 3 Mar 2022 22:12:31 +0000 Subject: [PATCH 276/519] chore(main): release 0.17.2 (#490) :robot: I have created a release *beep* *boop* --- ### [0.17.2](https://github.com/googleapis/python-bigquery-pandas/compare/v0.17.1...v0.17.2) (2022-03-02) ### Dependencies * allow pyarrow 7.0 ([#487](https://github.com/googleapis/python-bigquery-pandas/issues/487)) ([39441b6](https://github.com/googleapis/python-bigquery-pandas/commit/39441b63fadd95810c535e7079d781e9eec72189)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- packages/pandas-gbq/CHANGELOG.md | 7 +++++++ packages/pandas-gbq/pandas_gbq/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index 20f508762338..1411a5f9aa07 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +### [0.17.2](https://github.com/googleapis/python-bigquery-pandas/compare/v0.17.1...v0.17.2) (2022-03-02) + + +### Dependencies + +* allow pyarrow 7.0 ([#487](https://github.com/googleapis/python-bigquery-pandas/issues/487)) ([39441b6](https://github.com/googleapis/python-bigquery-pandas/commit/39441b63fadd95810c535e7079d781e9eec72189)) + ### [0.17.1](https://github.com/googleapis/python-bigquery-pandas/compare/v0.17.0...v0.17.1) (2022-02-24) diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py index e0370797b1f0..06e4348343a2 100644 --- a/packages/pandas-gbq/pandas_gbq/version.py +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -2,4 +2,4 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -__version__ = "0.17.1" +__version__ = "0.17.2" From 939508543c228a51ac0d27b36204e17318ed46c3 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 4 Mar 2022 12:56:49 -0500 Subject: [PATCH 277/519] chore: Adding support for pytest-xdist and pytest-parallel (#494) Source-Link: https://github.com/googleapis/synthtool/commit/82f5cb283efffe96e1b6cd634738e0e7de2cd90a Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:5d8da01438ece4021d135433f2cf3227aa39ef0eaccc941d62aa35e6902832ae Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 2 +- .../pandas-gbq/samples/snippets/noxfile.py | 78 +++++++++++-------- 2 files changed, 45 insertions(+), 35 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 480226ac08a9..7e08e05a380c 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:6162c384d685c5fe22521d3f37f6fc732bf99a085f6d47b677dbcae97fc21392 + digest: sha256:5d8da01438ece4021d135433f2cf3227aa39ef0eaccc941d62aa35e6902832ae diff --git a/packages/pandas-gbq/samples/snippets/noxfile.py b/packages/pandas-gbq/samples/snippets/noxfile.py index 20cdfc620138..85f5836dba3a 100644 --- a/packages/pandas-gbq/samples/snippets/noxfile.py +++ b/packages/pandas-gbq/samples/snippets/noxfile.py @@ -188,42 +188,52 @@ def _session_tests( # check for presence of tests test_list = glob.glob("*_test.py") + glob.glob("test_*.py") test_list.extend(glob.glob("tests")) + if len(test_list) == 0: print("No tests found, skipping directory.") - else: - if TEST_CONFIG["pip_version_override"]: - pip_version = TEST_CONFIG["pip_version_override"] - session.install(f"pip=={pip_version}") - """Runs py.test for a particular project.""" - if os.path.exists("requirements.txt"): - if os.path.exists("constraints.txt"): - session.install("-r", "requirements.txt", "-c", "constraints.txt") - else: - session.install("-r", "requirements.txt") - - if os.path.exists("requirements-test.txt"): - if os.path.exists("constraints-test.txt"): - session.install( - "-r", "requirements-test.txt", "-c", "constraints-test.txt" - ) - else: - session.install("-r", "requirements-test.txt") - - if INSTALL_LIBRARY_FROM_SOURCE: - session.install("-e", _get_repo_root()) - - if post_install: - post_install(session) - - session.run( - "pytest", - *(PYTEST_COMMON_ARGS + session.posargs), - # Pytest will return 5 when no tests are collected. This can happen - # on travis where slow and flaky tests are excluded. - # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html - success_codes=[0, 5], - env=get_pytest_env_vars(), - ) + return + + if TEST_CONFIG["pip_version_override"]: + pip_version = TEST_CONFIG["pip_version_override"] + session.install(f"pip=={pip_version}") + """Runs py.test for a particular project.""" + concurrent_args = [] + if os.path.exists("requirements.txt"): + if os.path.exists("constraints.txt"): + session.install("-r", "requirements.txt", "-c", "constraints.txt") + else: + session.install("-r", "requirements.txt") + with open("requirements.txt") as rfile: + packages = rfile.read() + + if os.path.exists("requirements-test.txt"): + if os.path.exists("constraints-test.txt"): + session.install("-r", "requirements-test.txt", "-c", "constraints-test.txt") + else: + session.install("-r", "requirements-test.txt") + with open("requirements-test.txt") as rtfile: + packages += rtfile.read() + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + if "pytest-parallel" in packages: + concurrent_args.extend(["--workers", "auto", "--tests-per-worker", "auto"]) + elif "pytest-xdist" in packages: + concurrent_args.extend(["-n", "auto"]) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs + concurrent_args), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars(), + ) @nox.session(python=ALL_VERSIONS) From 6ea98109f161b2aa489673726030619e7b514f1e Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 4 Mar 2022 13:59:42 -0500 Subject: [PATCH 278/519] fix(deps): require google-api-core>=1.31.5, >=2.3.2 (#493) * fix(deps): require google-api-core>=1.31.5, >=2.3.2 fix(deps): require proto-plus>=1.15.0 fix(deps): require google-auth>=1.25.0 --- packages/pandas-gbq/setup.py | 4 ++-- packages/pandas-gbq/testing/constraints-3.7.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index fd62b7200ae3..d395e71f81da 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -31,8 +31,8 @@ # Note: google-api-core and google-auth are also included via transitive # dependency on google-cloud-bigquery, but this library also uses them # directly. - "google-api-core >=1.21.0", - "google-auth >=1.18.0", + "google-api-core >= 1.31.5, <3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0", + "google-auth >=1.25.0", "google-auth-oauthlib >=0.0.1", # Require 1.27.* because it has a fix for out-of-bounds timestamps. See: # https://github.com/googleapis/python-bigquery/pull/209 and diff --git a/packages/pandas-gbq/testing/constraints-3.7.txt b/packages/pandas-gbq/testing/constraints-3.7.txt index f0c9a4acd0e4..2d4674eb6712 100644 --- a/packages/pandas-gbq/testing/constraints-3.7.txt +++ b/packages/pandas-gbq/testing/constraints-3.7.txt @@ -6,8 +6,8 @@ # e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", # Then this file should have foo==1.14.0 db-dtypes==0.3.1 -google-api-core==1.21.0 -google-auth==1.18.0 +google-api-core==1.31.5 +google-auth==1.25.0 google-auth-oauthlib==0.0.1 google-cloud-bigquery==1.27.2 google-cloud-bigquery-storage==1.1.0 From 74f5a26264173382a391a1252305c8a2421622f8 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 4 Mar 2022 19:41:51 -0500 Subject: [PATCH 279/519] chore(deps): update actions/download-artifact action to v3 (#497) Source-Link: https://github.com/googleapis/synthtool/commit/38e11ad1104dcc1e63b52691ddf2fe4015d06955 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:4e1991042fe54b991db9ca17c8fb386e61b22fe4d1472a568bf0fcac85dcf5d3 Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 2 +- packages/pandas-gbq/.github/workflows/unittest.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 7e08e05a380c..44c78f7cc12d 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:5d8da01438ece4021d135433f2cf3227aa39ef0eaccc941d62aa35e6902832ae + digest: sha256:4e1991042fe54b991db9ca17c8fb386e61b22fe4d1472a568bf0fcac85dcf5d3 diff --git a/packages/pandas-gbq/.github/workflows/unittest.yml b/packages/pandas-gbq/.github/workflows/unittest.yml index 0a86d57beb2a..c01a911337e9 100644 --- a/packages/pandas-gbq/.github/workflows/unittest.yml +++ b/packages/pandas-gbq/.github/workflows/unittest.yml @@ -26,7 +26,7 @@ jobs: run: | nox -s unit-${{ matrix.python }} - name: Upload coverage results - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: coverage-artifacts path: .coverage-${{ matrix.python }} @@ -47,7 +47,7 @@ jobs: python -m pip install --upgrade setuptools pip wheel python -m pip install coverage - name: Download coverage results - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: name: coverage-artifacts path: .coverage-results/ From dd94a4ef2da5df39a192c44cad39c33416f2b9a1 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 7 Mar 2022 13:01:49 -0600 Subject: [PATCH 280/519] chore(main): release 0.17.3 (#496) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- packages/pandas-gbq/CHANGELOG.md | 9 +++++++++ packages/pandas-gbq/pandas_gbq/version.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index 1411a5f9aa07..4a2c984c4b6e 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +### [0.17.3](https://github.com/googleapis/python-bigquery-pandas/compare/v0.17.2...v0.17.3) (2022-03-05) + + +### Bug Fixes + +* **deps:** require google-api-core>=1.31.5, >=2.3.2 ([#493](https://github.com/googleapis/python-bigquery-pandas/issues/493)) ([744a71c](https://github.com/googleapis/python-bigquery-pandas/commit/744a71c3d265d0e9a2ac25ca98dd0fa3ca68af6a)) +* **deps:** require google-auth>=1.25.0 ([744a71c](https://github.com/googleapis/python-bigquery-pandas/commit/744a71c3d265d0e9a2ac25ca98dd0fa3ca68af6a)) +* **deps:** require proto-plus>=1.15.0 ([744a71c](https://github.com/googleapis/python-bigquery-pandas/commit/744a71c3d265d0e9a2ac25ca98dd0fa3ca68af6a)) + ### [0.17.2](https://github.com/googleapis/python-bigquery-pandas/compare/v0.17.1...v0.17.2) (2022-03-02) diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py index 06e4348343a2..e51b9ce856b3 100644 --- a/packages/pandas-gbq/pandas_gbq/version.py +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -2,4 +2,4 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -__version__ = "0.17.2" +__version__ = "0.17.3" From c41d46ddb83ee6ac966549c68ad524a36bf3afc0 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 8 Mar 2022 13:53:41 -0600 Subject: [PATCH 281/519] fix: correctly transform query job timeout configuration and exceptions (#492) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: correctly transform query job timeout configuration and exceptions * unit tests for configuration transformations * add unit tests for timeout * link todo to relevant issue * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * use correct template for freezegun deps * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * decrease tick time in case multiple time calls are made * try no tick * add freezegun to conda deps * typo Co-authored-by: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> * typo Co-authored-by: Anthonios Partheniou Co-authored-by: Owl Bot Co-authored-by: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Co-authored-by: Anthonios Partheniou --- .../ci/requirements-3.7-0.24.2.conda | 1 + .../ci/requirements-3.9-1.3.4.conda | 1 + packages/pandas-gbq/noxfile.py | 1 + packages/pandas-gbq/owlbot.py | 1 + packages/pandas-gbq/pandas_gbq/gbq.py | 91 ++++++++++++++----- packages/pandas-gbq/tests/system/test_gbq.py | 17 +--- packages/pandas-gbq/tests/unit/test_gbq.py | 65 +++++++++++++ 7 files changed, 141 insertions(+), 36 deletions(-) diff --git a/packages/pandas-gbq/ci/requirements-3.7-0.24.2.conda b/packages/pandas-gbq/ci/requirements-3.7-0.24.2.conda index a99bd59e8763..2facfb2cd44d 100644 --- a/packages/pandas-gbq/ci/requirements-3.7-0.24.2.conda +++ b/packages/pandas-gbq/ci/requirements-3.7-0.24.2.conda @@ -3,6 +3,7 @@ coverage db-dtypes==0.3.1 fastavro flake8 +freezegun numpy==1.16.6 google-cloud-bigquery==1.27.2 google-cloud-bigquery-storage==1.1.0 diff --git a/packages/pandas-gbq/ci/requirements-3.9-1.3.4.conda b/packages/pandas-gbq/ci/requirements-3.9-1.3.4.conda index 73595253bdd6..1411fe5b06d7 100644 --- a/packages/pandas-gbq/ci/requirements-3.9-1.3.4.conda +++ b/packages/pandas-gbq/ci/requirements-3.9-1.3.4.conda @@ -3,6 +3,7 @@ coverage db-dtypes fastavro flake8 +freezegun google-cloud-bigquery google-cloud-bigquery-storage numpy diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 1b719448b518..209ec3aee6f9 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -94,6 +94,7 @@ def default(session): "-c", constraints_path, ) + session.install("freezegun", "-c", constraints_path) if session.python == "3.9": extras = "" diff --git a/packages/pandas-gbq/owlbot.py b/packages/pandas-gbq/owlbot.py index 5cf11d2ec940..02eeb0693a5c 100644 --- a/packages/pandas-gbq/owlbot.py +++ b/packages/pandas-gbq/owlbot.py @@ -38,6 +38,7 @@ unit_test_python_versions=["3.7", "3.8", "3.9", "3.10"], system_test_python_versions=["3.7", "3.8", "3.9", "3.10"], cov_level=96, + unit_test_external_dependencies=["freezegun"], unit_test_extras=extras, unit_test_extras_by_python=extras_by_python, system_test_extras=extras, diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 6c9b6804f080..6d06d3d63d94 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -2,6 +2,8 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. +import copy +import concurrent.futures from datetime import datetime import logging import re @@ -378,6 +380,9 @@ def process_http_error(ex): # See `BigQuery Troubleshooting Errors # `__ + if "cancelled" in ex.message: + raise QueryTimeout("Reason: {0}".format(ex)) + raise GenericGBQException("Reason: {0}".format(ex)) def download_table( @@ -406,8 +411,41 @@ def download_table( user_dtypes=dtypes, ) + def _wait_for_query_job(self, query_reply, timeout_ms): + """Wait for query to complete, pausing occasionally to update progress. + + Args: + query_reply (QueryJob): + A query job which has started. + + timeout_ms (Optional[int]): + How long to wait before cancelling the query. + """ + # Wait at most 10 seconds so we can show progress. + # TODO(https://github.com/googleapis/python-bigquery-pandas/issues/327): + # Include a tqdm progress bar here instead of a stream of log messages. + timeout_sec = 10.0 + if timeout_ms: + timeout_sec = min(timeout_sec, timeout_ms / 1000.0) + + while query_reply.state != "DONE": + self.log_elapsed_seconds(" Elapsed", "s. Waiting...") + + if timeout_ms and timeout_ms < self.get_elapsed_seconds() * 1000: + self.client.cancel_job( + query_reply.job_id, location=query_reply.location + ) + raise QueryTimeout("Query timeout: {} ms".format(timeout_ms)) + + try: + query_reply.result(timeout=timeout_sec) + except concurrent.futures.TimeoutError: + # Use our own timeout logic + pass + except self.http_error as ex: + self.process_http_error(ex) + def run_query(self, query, max_results=None, progress_bar_type=None, **kwargs): - from concurrent.futures import TimeoutError from google.auth.exceptions import RefreshError from google.cloud import bigquery import pandas @@ -449,28 +487,11 @@ def run_query(self, query, max_results=None, progress_bar_type=None, **kwargs): job_id = query_reply.job_id logger.debug("Job ID: %s" % job_id) - while query_reply.state != "DONE": - self.log_elapsed_seconds(" Elapsed", "s. Waiting...") - - timeout_ms = job_config.get("jobTimeoutMs") or job_config["query"].get( - "timeoutMs" - ) - timeout_ms = int(timeout_ms) if timeout_ms else None - if timeout_ms and timeout_ms < self.get_elapsed_seconds() * 1000: - raise QueryTimeout("Query timeout: {} ms".format(timeout_ms)) - - timeout_sec = 1.0 - if timeout_ms: - # Wait at most 1 second so we can show progress bar - timeout_sec = min(1.0, timeout_ms / 1000.0) - - try: - query_reply.result(timeout=timeout_sec) - except TimeoutError: - # Use our own timeout logic - pass - except self.http_error as ex: - self.process_http_error(ex) + timeout_ms = job_config.get("jobTimeoutMs") or job_config["query"].get( + "timeoutMs" + ) + timeout_ms = int(timeout_ms) if timeout_ms else None + self._wait_for_query_job(query_reply, timeout_ms) if query_reply.cache_hit: logger.debug("Query done.\nCache hit.\n") @@ -673,6 +694,28 @@ def _finalize_dtypes( return df +def _transform_read_gbq_configuration(configuration): + """ + For backwards-compatibility, convert any previously client-side only + parameters such as timeoutMs to the property name expected by the REST API. + + Makes a copy of configuration if changes are needed. + """ + + if configuration is None: + return None + + timeout_ms = configuration.get("query", {}).get("timeoutMs") + if timeout_ms is not None: + # Transform timeoutMs to an actual server-side configuration. + # https://github.com/googleapis/python-bigquery-pandas/issues/479 + configuration = copy.deepcopy(configuration) + del configuration["query"]["timeoutMs"] + configuration["jobTimeoutMs"] = timeout_ms + + return configuration + + def read_gbq( query_or_table, project_id=None, @@ -847,6 +890,8 @@ def read_gbq( if dialect not in ("legacy", "standard"): raise ValueError("'{0}' is not valid for dialect".format(dialect)) + configuration = _transform_read_gbq_configuration(configuration) + if configuration and "query" in configuration and "query" in configuration["query"]: if query_or_table is not None: raise ValueError( diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index 214b1f7457fa..2290744c1fda 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -473,22 +473,13 @@ def test_configuration_raises_value_error_with_multiple_config(self, project_id) def test_timeout_configuration(self, project_id): sql_statement = """ - SELECT - SUM(bottles_sold) total_bottles, - UPPER(category_name) category_name, - magnitude, - liquor.zip_code zip_code - FROM `bigquery-public-data.iowa_liquor_sales.sales` liquor - JOIN `bigquery-public-data.geo_us_boundaries.zip_codes` zip_codes - ON liquor.zip_code = zip_codes.zip_code - JOIN `bigquery-public-data.noaa_historic_severe_storms.tornado_paths` tornados - ON liquor.date = tornados.storm_date - WHERE ST_INTERSECTS(tornado_path_geom, zip_code_geom) - GROUP BY category_name, magnitude, zip_code - ORDER BY magnitude ASC, total_bottles DESC + select count(*) from unnest(generate_array(1,1000000)), unnest(generate_array(1, 10000)) """ configs = [ + # pandas-gbq timeout configuration. Transformed to REST API compatible version. {"query": {"useQueryCache": False, "timeoutMs": 1}}, + # REST API job timeout. See: + # https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfiguration.FIELDS.job_timeout_ms {"query": {"useQueryCache": False}, "jobTimeoutMs": 1}, ] for config in configs: diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index 511e68d6404b..457d356bb9e6 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -4,10 +4,12 @@ # -*- coding: utf-8 -*- +import concurrent.futures import copy import datetime from unittest import mock +import freezegun import google.api_core.exceptions import numpy import pandas @@ -114,6 +116,61 @@ def test__is_query(query_or_table, expected): assert result == expected +@pytest.mark.parametrize( + ["original", "expected"], + [ + (None, None), + ({}, {}), + ({"query": {"useQueryCache": False}}, {"query": {"useQueryCache": False}}), + ({"jobTimeoutMs": "1234"}, {"jobTimeoutMs": "1234"}), + ({"query": {"timeoutMs": "1234"}}, {"query": {}, "jobTimeoutMs": "1234"}), + ], +) +def test__transform_read_gbq_configuration_makes_copy(original, expected): + should_change = original == expected + got = gbq._transform_read_gbq_configuration(original) + assert got == expected + # Catch if we accidentally modified the original. + did_change = original == got + assert did_change == should_change + + +def test__wait_for_query_job_exits_when_done(mock_bigquery_client): + connector = _make_connector() + connector.client = mock_bigquery_client + connector.start = datetime.datetime(2020, 1, 1).timestamp() + + mock_query = mock.create_autospec(google.cloud.bigquery.QueryJob) + type(mock_query).state = mock.PropertyMock(side_effect=("RUNNING", "DONE")) + mock_query.result.side_effect = concurrent.futures.TimeoutError("fake timeout") + + with freezegun.freeze_time("2020-01-01 00:00:00", tick=False): + connector._wait_for_query_job(mock_query, 60) + + mock_bigquery_client.cancel_job.assert_not_called() + + +def test__wait_for_query_job_cancels_after_timeout(mock_bigquery_client): + connector = _make_connector() + connector.client = mock_bigquery_client + connector.start = datetime.datetime(2020, 1, 1).timestamp() + + mock_query = mock.create_autospec(google.cloud.bigquery.QueryJob) + mock_query.job_id = "a-random-id" + mock_query.location = "job-location" + mock_query.state = "RUNNING" + mock_query.result.side_effect = concurrent.futures.TimeoutError("fake timeout") + + with freezegun.freeze_time( + "2020-01-01 00:00:00", auto_tick_seconds=15 + ), pytest.raises(gbq.QueryTimeout): + connector._wait_for_query_job(mock_query, 60) + + mock_bigquery_client.cancel_job.assert_called_with( + "a-random-id", location="job-location" + ) + + def test_GbqConnector_get_client_w_new_bq(mock_bigquery_client): gbq._test_google_api_imports() pytest.importorskip("google.api_core.client_info") @@ -125,6 +182,14 @@ def test_GbqConnector_get_client_w_new_bq(mock_bigquery_client): assert kwargs["client_info"].user_agent == "pandas-{}".format(pandas.__version__) +def test_GbqConnector_process_http_error_transforms_timeout(): + original = google.api_core.exceptions.GoogleAPICallError( + "Job execution was cancelled: Job timed out after 0s" + ) + with pytest.raises(gbq.QueryTimeout): + gbq.GbqConnector.process_http_error(original) + + def test_to_gbq_should_fail_if_invalid_table_name_passed(): with pytest.raises(gbq.NotFoundException): gbq.to_gbq(DataFrame([[1]]), "invalid_table_name", project_id="1234") From 120f0ba2d41d3c640a941e2a808741cccc481b04 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 14 Mar 2022 12:10:14 -0500 Subject: [PATCH 282/519] fix: avoid deprecated "out-of-band" authentication flow (#500) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The console-based copy-paste-a-token flow has been deprecated. See: https://developers.googleblog.com/2022/02/making-oauth-flows-safer.html?m=1#disallowed-oob Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - [ ] Make sure to open an issue as a [bug/issue](https://github.com/googleapis/python-bigquery-pandas/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea - [ ] Ensure the tests and linter pass - [ ] Code coverage does not decrease (if any source code was changed) - [ ] Appropriate docs were updated (if necessary) Fixes # 🦕 --- packages/pandas-gbq/docs/howto/authentication.rst | 8 +++++--- packages/pandas-gbq/pandas_gbq/auth.py | 2 +- packages/pandas-gbq/pandas_gbq/gbq.py | 10 +++++----- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/packages/pandas-gbq/docs/howto/authentication.rst b/packages/pandas-gbq/docs/howto/authentication.rst index 877b11894ee2..029d5f38a5c7 100644 --- a/packages/pandas-gbq/docs/howto/authentication.rst +++ b/packages/pandas-gbq/docs/howto/authentication.rst @@ -156,9 +156,11 @@ credentials are not found. credentials = pydata_google_auth.get_user_credentials( SCOPES, - # Set auth_local_webserver to True to have a slightly more convienient - # authorization flow. Note, this doesn't work if you're running from a - # notebook on a remote sever, such as over SSH or with Google Colab. + # Note, this doesn't work if you're running from a notebook on a + # remote sever, such as over SSH or with Google Colab. In those cases, + # install the gcloud command line interface and authenticate with the + # `gcloud auth application-default login` command and the `--no-browser` + # option. auth_local_webserver=True, ) diff --git a/packages/pandas-gbq/pandas_gbq/auth.py b/packages/pandas-gbq/pandas_gbq/auth.py index 41ee4192f573..d59f75f34745 100644 --- a/packages/pandas-gbq/pandas_gbq/auth.py +++ b/packages/pandas-gbq/pandas_gbq/auth.py @@ -28,7 +28,7 @@ def get_credentials( - private_key=None, project_id=None, reauth=False, auth_local_webserver=False + private_key=None, project_id=None, reauth=False, auth_local_webserver=True ): import pydata_google_auth diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 6d06d3d63d94..41456d799c19 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -281,7 +281,7 @@ def __init__( project_id, reauth=False, private_key=None, - auth_local_webserver=False, + auth_local_webserver=True, dialect="standard", location=None, credentials=None, @@ -722,7 +722,7 @@ def read_gbq( index_col=None, col_order=None, reauth=False, - auth_local_webserver=False, + auth_local_webserver=True, dialect=None, location=None, configuration=None, @@ -762,7 +762,7 @@ def read_gbq( reauth : boolean, default False Force Google BigQuery to re-authenticate the user. This is useful if multiple accounts are used. - auth_local_webserver : bool, default False + auth_local_webserver : bool, default True Use the `local webserver flow `_ instead of the `console flow @@ -959,7 +959,7 @@ def to_gbq( chunksize=None, reauth=False, if_exists="fail", - auth_local_webserver=False, + auth_local_webserver=True, table_schema=None, location=None, progress_bar=True, @@ -1005,7 +1005,7 @@ def to_gbq( If table exists, drop it, recreate it, and insert data. ``'append'`` If table exists, insert data. Create if does not exist. - auth_local_webserver : bool, default False + auth_local_webserver : bool, default True Use the `local webserver flow `_ instead of the `console flow From a718cbf9d21f6c4bdd6fea7bfab7f0a69ff6bb48 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 14 Mar 2022 18:26:16 +0000 Subject: [PATCH 283/519] chore(main): release 0.17.4 (#499) :robot: I have created a release *beep* *boop* --- ### [0.17.4](https://github.com/googleapis/python-bigquery-pandas/compare/v0.17.3...v0.17.4) (2022-03-14) ### Bug Fixes * avoid deprecated "out-of-band" authentication flow ([#500](https://github.com/googleapis/python-bigquery-pandas/issues/500)) ([4758e3a](https://github.com/googleapis/python-bigquery-pandas/commit/4758e3a9ccb82109aae65f76258b2910077e02dd)) * correctly transform query job timeout configuration and exceptions ([#492](https://github.com/googleapis/python-bigquery-pandas/issues/492)) ([d8c3900](https://github.com/googleapis/python-bigquery-pandas/commit/d8c3900eda5aa2cb5b663b2be569d639f6a028a9)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- packages/pandas-gbq/CHANGELOG.md | 8 ++++++++ packages/pandas-gbq/pandas_gbq/version.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index 4a2c984c4b6e..d94f687f33c7 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +### [0.17.4](https://github.com/googleapis/python-bigquery-pandas/compare/v0.17.3...v0.17.4) (2022-03-14) + + +### Bug Fixes + +* avoid deprecated "out-of-band" authentication flow ([#500](https://github.com/googleapis/python-bigquery-pandas/issues/500)) ([4758e3a](https://github.com/googleapis/python-bigquery-pandas/commit/4758e3a9ccb82109aae65f76258b2910077e02dd)) +* correctly transform query job timeout configuration and exceptions ([#492](https://github.com/googleapis/python-bigquery-pandas/issues/492)) ([d8c3900](https://github.com/googleapis/python-bigquery-pandas/commit/d8c3900eda5aa2cb5b663b2be569d639f6a028a9)) + ### [0.17.3](https://github.com/googleapis/python-bigquery-pandas/compare/v0.17.2...v0.17.3) (2022-03-05) diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py index e51b9ce856b3..862c78c93c7f 100644 --- a/packages/pandas-gbq/pandas_gbq/version.py +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -2,4 +2,4 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -__version__ = "0.17.3" +__version__ = "0.17.4" From 7ee5b181bb1ec661b79aebe92aad6f6b4597bd3e Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Mon, 21 Mar 2022 17:29:43 +0100 Subject: [PATCH 284/519] chore(deps): update all dependencies (#498) * chore(deps): update all dependencies * revert change Co-authored-by: Anthonios Partheniou --- packages/pandas-gbq/samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 254e656f444b..495b15c33e21 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,5 +1,5 @@ google-cloud-bigquery-storage==2.12.0 -google-cloud-bigquery==2.34.0 +google-cloud-bigquery==2.34.1 pandas==1.3.5; python_version < '3.8' pandas==1.4.1; python_version >= '3.8' pyarrow==7.0.0 From b0df9a678f6803ebf5dd0256d36f6087533b335f Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Wed, 23 Mar 2022 16:43:16 +0100 Subject: [PATCH 285/519] chore(deps): update all dependencies (#502) * chore(deps): update all dependencies * revert change Co-authored-by: Anthonios Partheniou --- packages/pandas-gbq/samples/snippets/requirements-test.txt | 2 +- packages/pandas-gbq/samples/snippets/requirements.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements-test.txt b/packages/pandas-gbq/samples/snippets/requirements-test.txt index 048b49808a9d..e227954a1e25 100644 --- a/packages/pandas-gbq/samples/snippets/requirements-test.txt +++ b/packages/pandas-gbq/samples/snippets/requirements-test.txt @@ -1,2 +1,2 @@ google-cloud-testutils==1.3.1 -pytest==7.0.1 +pytest==7.1.1 diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 495b15c33e21..0cfd8f264919 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,5 +1,5 @@ -google-cloud-bigquery-storage==2.12.0 -google-cloud-bigquery==2.34.1 +google-cloud-bigquery-storage==2.13.0 +google-cloud-bigquery==2.34.2 pandas==1.3.5; python_version < '3.8' pandas==1.4.1; python_version >= '3.8' pyarrow==7.0.0 From 5ee56a18587eab2b6966bf65d71662111c130c37 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 29 Mar 2022 00:08:10 +0000 Subject: [PATCH 286/519] chore(python): use black==22.3.0 (#506) Source-Link: https://github.com/googleapis/synthtool/commit/6fab84af09f2cf89a031fd8671d1def6b2931b11 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:7cffbc10910c3ab1b852c05114a08d374c195a81cdec1d4a67a1d129331d0bfe --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 2 +- packages/pandas-gbq/docs/conf.py | 23 ++++++++-- packages/pandas-gbq/noxfile.py | 9 ++-- packages/pandas-gbq/pandas_gbq/auth.py | 3 +- packages/pandas-gbq/pandas_gbq/gbq.py | 15 +++++-- packages/pandas-gbq/pandas_gbq/load.py | 3 +- .../pandas-gbq/samples/snippets/noxfile.py | 4 +- packages/pandas-gbq/tests/system/test_gbq.py | 16 ++++--- .../pandas-gbq/tests/system/test_read_gbq.py | 43 ++++++++++++++----- .../system/test_read_gbq_with_bqstorage.py | 5 ++- .../pandas-gbq/tests/system/test_to_gbq.py | 11 +++-- packages/pandas-gbq/tests/unit/test_gbq.py | 40 +++++++++++------ packages/pandas-gbq/tests/unit/test_load.py | 14 +++--- packages/pandas-gbq/tests/unit/test_schema.py | 14 ++++-- packages/pandas-gbq/tests/unit/test_to_gbq.py | 4 +- 15 files changed, 146 insertions(+), 60 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 44c78f7cc12d..87dd00611576 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:4e1991042fe54b991db9ca17c8fb386e61b22fe4d1472a568bf0fcac85dcf5d3 + digest: sha256:7cffbc10910c3ab1b852c05114a08d374c195a81cdec1d4a67a1d129331d0bfe diff --git a/packages/pandas-gbq/docs/conf.py b/packages/pandas-gbq/docs/conf.py index a4eed21b11c6..ef544e748be5 100644 --- a/packages/pandas-gbq/docs/conf.py +++ b/packages/pandas-gbq/docs/conf.py @@ -279,7 +279,13 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (root_doc, "pandas-gbq.tex", "pandas-gbq Documentation", author, "manual",) + ( + root_doc, + "pandas-gbq.tex", + "pandas-gbq Documentation", + author, + "manual", + ) ] # The name of an image file (relative to this directory) to place at the top of @@ -307,7 +313,15 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [(root_doc, "pandas-gbq", "pandas-gbq Documentation", [author], 1,)] +man_pages = [ + ( + root_doc, + "pandas-gbq", + "pandas-gbq Documentation", + [author], + 1, + ) +] # If true, show URL addresses after external links. # man_show_urls = False @@ -347,7 +361,10 @@ intersphinx_mapping = { "python": ("https://python.readthedocs.org/en/latest/", None), "google-auth": ("https://googleapis.dev/python/google-auth/latest/", None), - "google.api_core": ("https://googleapis.dev/python/google-api-core/latest/", None,), + "google.api_core": ( + "https://googleapis.dev/python/google-api-core/latest/", + None, + ), "grpc": ("https://grpc.github.io/grpc/python/", None), "proto-plus": ("https://proto-plus-python.readthedocs.io/en/latest/", None), "protobuf": ("https://googleapis.dev/python/protobuf/latest/", None), diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 209ec3aee6f9..11bb1c90b75c 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -25,7 +25,7 @@ import nox -BLACK_VERSION = "black==19.10b0" +BLACK_VERSION = "black==22.3.0" BLACK_PATHS = ["docs", "pandas_gbq", "tests", "noxfile.py", "setup.py"] DEFAULT_PYTHON_VERSION = "3.8" @@ -58,7 +58,9 @@ def lint(session): """ session.install("flake8", BLACK_VERSION) session.run( - "black", "--check", *BLACK_PATHS, + "black", + "--check", + *BLACK_PATHS, ) session.run("flake8", "pandas_gbq", "tests") @@ -68,7 +70,8 @@ def blacken(session): """Run black. Format code to uniform standard.""" session.install(BLACK_VERSION) session.run( - "black", *BLACK_PATHS, + "black", + *BLACK_PATHS, ) diff --git a/packages/pandas-gbq/pandas_gbq/auth.py b/packages/pandas-gbq/pandas_gbq/auth.py index d59f75f34745..47f989b517f9 100644 --- a/packages/pandas-gbq/pandas_gbq/auth.py +++ b/packages/pandas-gbq/pandas_gbq/auth.py @@ -58,7 +58,8 @@ def get_credentials_cache(reauth): if reauth: return pydata_google_auth.cache.WriteOnlyCredentialsCache( - dirname=CREDENTIALS_CACHE_DIRNAME, filename=CREDENTIALS_CACHE_FILENAME, + dirname=CREDENTIALS_CACHE_DIRNAME, + filename=CREDENTIALS_CACHE_FILENAME, ) return pydata_google_auth.cache.ReadWriteCredentialsCache( dirname=CREDENTIALS_CACHE_DIRNAME, filename=CREDENTIALS_CACHE_FILENAME diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 41456d799c19..56d6fd706807 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -338,7 +338,7 @@ def __init__( # BQ Queries costs $5 per TB. First 1 TB per month is free # see here for more: https://cloud.google.com/bigquery/pricing - self.query_price_for_TB = 5.0 / 2 ** 40 # USD/TB + self.query_price_for_TB = 5.0 / 2**40 # USD/TB def _start_timer(self): self.start = time.time() @@ -500,7 +500,8 @@ def run_query(self, query, max_results=None, progress_bar_type=None, **kwargs): bytes_billed = query_reply.total_bytes_billed or 0 logger.debug( "Query done.\nProcessed: {} Billed: {}".format( - self.sizeof_fmt(bytes_processed), self.sizeof_fmt(bytes_billed), + self.sizeof_fmt(bytes_processed), + self.sizeof_fmt(bytes_billed), ) ) logger.debug( @@ -533,7 +534,11 @@ def run_query(self, query, max_results=None, progress_bar_type=None, **kwargs): ) def _download_results( - self, rows_iter, max_results=None, progress_bar_type=None, user_dtypes=None, + self, + rows_iter, + max_results=None, + progress_bar_type=None, + user_dtypes=None, ): # No results are desired, so don't bother downloading anything. if max_results == 0: @@ -1309,7 +1314,9 @@ def create(self, table_id, schema): self.dataset_id ): _Dataset( - self.project_id, credentials=self.credentials, location=self.location, + self.project_id, + credentials=self.credentials, + location=self.location, ).create(self.dataset_id) table_ref = TableReference( diff --git a/packages/pandas-gbq/pandas_gbq/load.py b/packages/pandas-gbq/pandas_gbq/load.py index e52952f2501b..2fcf26109bf0 100644 --- a/packages/pandas-gbq/pandas_gbq/load.py +++ b/packages/pandas-gbq/pandas_gbq/load.py @@ -59,7 +59,8 @@ def split_dataframe(dataframe, chunksize=None): def cast_dataframe_for_parquet( - dataframe: pandas.DataFrame, schema: Optional[Dict[str, Any]], + dataframe: pandas.DataFrame, + schema: Optional[Dict[str, Any]], ) -> pandas.DataFrame: """Cast columns to needed dtype when writing parquet files. diff --git a/packages/pandas-gbq/samples/snippets/noxfile.py b/packages/pandas-gbq/samples/snippets/noxfile.py index 85f5836dba3a..25f87a215d4c 100644 --- a/packages/pandas-gbq/samples/snippets/noxfile.py +++ b/packages/pandas-gbq/samples/snippets/noxfile.py @@ -29,7 +29,7 @@ # WARNING - WARNING - WARNING - WARNING - WARNING # WARNING - WARNING - WARNING - WARNING - WARNING -BLACK_VERSION = "black==19.10b0" +BLACK_VERSION = "black==22.3.0" # Copy `noxfile_config.py` to your directory and modify it instead. @@ -253,7 +253,7 @@ def py(session: nox.sessions.Session) -> None: def _get_repo_root() -> Optional[str]: - """ Returns the root folder of the project. """ + """Returns the root folder of the project.""" # Get root of this repository. Assume we don't have directories nested deeper than 10 items. p = Path(os.getcwd()) for i in range(10): diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index 2290744c1fda..ee8190b511f7 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -87,7 +87,8 @@ def get_schema(gbq_connector: gbq.GbqConnector, dataset_id: str, table_id: str): bqclient = gbq_connector.client table_ref = bigquery.TableReference( - bigquery.DatasetReference(bqclient.project, dataset_id), table_id, + bigquery.DatasetReference(bqclient.project, dataset_id), + table_id, ) try: @@ -518,7 +519,8 @@ def test_array(self, project_id): dialect="standard", ) tm.assert_frame_equal( - df, DataFrame([[["a", "x", "b", "y", "c", "z"]]], columns=["letters"]), + df, + DataFrame([[["a", "x", "b", "y", "c", "z"]]], columns=["letters"]), ) def test_array_length_zero(self, project_id): @@ -565,7 +567,8 @@ def test_array_agg(self, project_id): dialect="standard", ) tm.assert_frame_equal( - df, DataFrame([["a", [1, 3]], ["b", [2]]], columns=["letter", "numbers"]), + df, + DataFrame([["a", [1, 3]], ["b", [2]]], columns=["letter", "numbers"]), ) def test_array_of_floats(self, project_id): @@ -990,7 +993,9 @@ def test_upload_data_with_timestamp(self, project_id): test_id = "21" test_size = 6 df = DataFrame( - np.random.randn(test_size, 4), index=range(test_size), columns=list("ABCD"), + np.random.randn(test_size, 4), + index=range(test_size), + columns=list("ABCD"), ) df["times"] = pandas.Series( [ @@ -1069,7 +1074,8 @@ def test_upload_data_tokyo(self, project_id, tokyo_dataset, bigquery_client): table = bigquery_client.get_table( bigquery.TableReference( - bigquery.DatasetReference(project_id, tokyo_dataset), "to_gbq_test", + bigquery.DatasetReference(project_id, tokyo_dataset), + "to_gbq_test", ) ) assert table.num_rows > 0 diff --git a/packages/pandas-gbq/tests/system/test_read_gbq.py b/packages/pandas-gbq/tests/system/test_read_gbq.py index 65a65ff73e0a..0187b69d82e4 100644 --- a/packages/pandas-gbq/tests/system/test_read_gbq.py +++ b/packages/pandas-gbq/tests/system/test_read_gbq.py @@ -153,7 +153,7 @@ def writable_table( ), "float_col": [1.125, -2.375, 0.0], "int64_col": pandas.Series( - [(2 ** 63) - 1, -1, -(2 ** 63)], dtype="Int64" + [(2**63) - 1, -1, -(2**63)], dtype="Int64" ), "numeric_col": [ decimal.Decimal("123.456789"), @@ -377,15 +377,25 @@ def writable_table( else "object", ), "bytes_col": [None], - "date_col": pandas.Series([None], dtype=db_dtypes.DateDtype(),), - "datetime_col": pandas.Series([None], dtype="datetime64[ns]",), + "date_col": pandas.Series( + [None], + dtype=db_dtypes.DateDtype(), + ), + "datetime_col": pandas.Series( + [None], + dtype="datetime64[ns]", + ), "float_col": pandas.Series([None], dtype="float64"), "int64_col": pandas.Series([None], dtype="Int64"), "numeric_col": [None], "string_col": [None], - "time_col": pandas.Series([None], dtype=db_dtypes.TimeDtype(),), + "time_col": pandas.Series( + [None], + dtype=db_dtypes.TimeDtype(), + ), "timestamp_col": pandas.Series( - [None], dtype="datetime64[ns]", + [None], + dtype="datetime64[ns]", ).dt.tz_localize(datetime.timezone.utc), } ), @@ -605,19 +615,30 @@ def test_empty_dataframe(read_gbq, use_bqstorage_api): { "row_num": pandas.Series([], dtype="Int64"), "bool_col": pandas.Series( - [], dtype="boolean" if FEATURES.pandas_has_boolean_dtype else "bool", + [], + dtype="boolean" if FEATURES.pandas_has_boolean_dtype else "bool", ), "bytes_col": pandas.Series([], dtype="object"), - "date_col": pandas.Series([], dtype=db_dtypes.DateDtype(),), - "datetime_col": pandas.Series([], dtype="datetime64[ns]",), + "date_col": pandas.Series( + [], + dtype=db_dtypes.DateDtype(), + ), + "datetime_col": pandas.Series( + [], + dtype="datetime64[ns]", + ), "float_col": pandas.Series([], dtype="float64"), "int64_col": pandas.Series([], dtype="Int64"), "numeric_col": pandas.Series([], dtype="object"), "string_col": pandas.Series([], dtype="object"), - "time_col": pandas.Series([], dtype=db_dtypes.TimeDtype(),), - "timestamp_col": pandas.Series([], dtype="datetime64[ns]",).dt.tz_localize( - datetime.timezone.utc + "time_col": pandas.Series( + [], + dtype=db_dtypes.TimeDtype(), ), + "timestamp_col": pandas.Series( + [], + dtype="datetime64[ns]", + ).dt.tz_localize(datetime.timezone.utc), } ) result = read_gbq(query, use_bqstorage_api=use_bqstorage_api) diff --git a/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py b/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py index cddcedf0260f..84435b9ff7ce 100644 --- a/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py +++ b/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py @@ -36,7 +36,10 @@ def test_empty_results(method_under_test, query_string): See: https://github.com/pydata/pandas-gbq/issues/299 """ - df = method_under_test(query_string, use_bqstorage_api=True,) + df = method_under_test( + query_string, + use_bqstorage_api=True, + ) assert len(df.index) == 0 diff --git a/packages/pandas-gbq/tests/system/test_to_gbq.py b/packages/pandas-gbq/tests/system/test_to_gbq.py index ae3b8614b9e7..a03113d7724e 100644 --- a/packages/pandas-gbq/tests/system/test_to_gbq.py +++ b/packages/pandas-gbq/tests/system/test_to_gbq.py @@ -112,7 +112,10 @@ def test_series_round_trip( round_trip = read_gbq(table_id) round_trip_series = round_trip["test_col"].sort_values().reset_index(drop=True) pandas.testing.assert_series_equal( - round_trip_series, input_series, check_exact=True, check_names=False, + round_trip_series, + input_series, + check_exact=True, + check_names=False, ) @@ -131,7 +134,8 @@ def test_series_round_trip( { "row_num": [0, 1, 2], "date_col": pandas.Series( - ["2021-04-17", "1999-12-31", "2038-01-19"], dtype="datetime64[ns]", + ["2021-04-17", "1999-12-31", "2038-01-19"], + dtype="datetime64[ns]", ), } ), @@ -177,7 +181,8 @@ def test_series_round_trip( # https://github.com/googleapis/python-bigquery-pandas/issues/421 DataFrameRoundTripTestCase( input_df=pandas.DataFrame( - {"row_num": [123], "num_col": [1.25]}, columns=["row_num", "num_col"], + {"row_num": [123], "num_col": [1.25]}, + columns=["row_num", "num_col"], ), expected_df=pandas.DataFrame( {"row_num": [123], "num_col": [decimal.Decimal("1.25")]}, diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index 457d356bb9e6..5184562ab293 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -339,7 +339,9 @@ def test_to_gbq_w_project_table(mock_bigquery_client): "my_table" ) gbq.to_gbq( - DataFrame(), "project_table.my_dataset.my_table", project_id="project_client", + DataFrame(), + "project_table.my_dataset.my_table", + project_id="project_client", ) mock_bigquery_client.get_table.assert_called_with( @@ -390,8 +392,8 @@ def test_dataset_exists_true(mock_bigquery_client): def test_dataset_exists_translates_exception(mock_bigquery_client): connector = gbq._Dataset("my-project") connector.client = mock_bigquery_client - mock_bigquery_client.get_dataset.side_effect = google.api_core.exceptions.InternalServerError( - "something went wrong" + mock_bigquery_client.get_dataset.side_effect = ( + google.api_core.exceptions.InternalServerError("something went wrong") ) with pytest.raises(gbq.GenericGBQException): connector.exists("not_gonna_work") @@ -413,8 +415,8 @@ def test_table_create_translates_exception(mock_bigquery_client): mock_bigquery_client.get_table.side_effect = google.api_core.exceptions.NotFound( "nope" ) - mock_bigquery_client.create_table.side_effect = google.api_core.exceptions.InternalServerError( - "something went wrong" + mock_bigquery_client.create_table.side_effect = ( + google.api_core.exceptions.InternalServerError("something went wrong") ) with pytest.raises(gbq.GenericGBQException): connector.create( @@ -435,8 +437,8 @@ def test_table_delete_notfound_ok(mock_bigquery_client): def test_table_delete_translates_exception(mock_bigquery_client): connector = gbq._Table("my-project", "my_dataset") connector.client = mock_bigquery_client - mock_bigquery_client.delete_table.side_effect = google.api_core.exceptions.InternalServerError( - "something went wrong" + mock_bigquery_client.delete_table.side_effect = ( + google.api_core.exceptions.InternalServerError("something went wrong") ) with pytest.raises(gbq.GenericGBQException): connector.delete("not_gonna_work") @@ -461,8 +463,8 @@ def test_table_exists_true(mock_bigquery_client): def test_table_exists_translates_exception(mock_bigquery_client): connector = gbq._Table("my-project", "my_dataset") connector.client = mock_bigquery_client - mock_bigquery_client.get_table.side_effect = google.api_core.exceptions.InternalServerError( - "something went wrong" + mock_bigquery_client.get_table.side_effect = ( + google.api_core.exceptions.InternalServerError("something went wrong") ) with pytest.raises(gbq.GenericGBQException): connector.exists("not_gonna_work") @@ -488,7 +490,9 @@ def test_read_gbq_with_inferred_project_id_from_service_account_credentials( ): mock_service_account_credentials.project_id = "service_account_project_id" df = gbq.read_gbq( - "SELECT 1", dialect="standard", credentials=mock_service_account_credentials, + "SELECT 1", + dialect="standard", + credentials=mock_service_account_credentials, ) assert df is not None mock_bigquery_client.query.assert_called_once_with( @@ -504,7 +508,9 @@ def test_read_gbq_without_inferred_project_id_from_compute_engine_credentials( ): with pytest.raises(ValueError, match="Could not determine project ID"): gbq.read_gbq( - "SELECT 1", dialect="standard", credentials=mock_compute_engine_credentials, + "SELECT 1", + dialect="standard", + credentials=mock_compute_engine_credentials, ) @@ -547,7 +553,8 @@ def test_read_gbq_with_old_bq_raises_importerror(monkeypatch): monkeypatch.setattr(FEATURES, "_bigquery_installed_version", None) with pytest.raises(ImportError, match="google-cloud-bigquery"): gbq.read_gbq( - "SELECT 1", project_id="my-project", + "SELECT 1", + project_id="my-project", ) @@ -558,7 +565,10 @@ def test_read_gbq_with_verbose_old_pandas_no_warnings(monkeypatch, recwarn): mock.PropertyMock(return_value=False), ) gbq.read_gbq( - "SELECT 1", project_id="my-project", dialect="standard", verbose=True, + "SELECT 1", + project_id="my-project", + dialect="standard", + verbose=True, ) assert len(recwarn) == 0 @@ -678,7 +688,9 @@ def test_read_gbq_use_bqstorage_api( ) else: mock_list_rows.to_dataframe.assert_called_once_with( - create_bqstorage_client=True, dtypes=mock.ANY, progress_bar_type=mock.ANY, + create_bqstorage_client=True, + dtypes=mock.ANY, + progress_bar_type=mock.ANY, ) diff --git a/packages/pandas-gbq/tests/unit/test_load.py b/packages/pandas-gbq/tests/unit/test_load.py index 24f262e6ab10..f2209bdac1f7 100644 --- a/packages/pandas-gbq/tests/unit/test_load.py +++ b/packages/pandas-gbq/tests/unit/test_load.py @@ -35,11 +35,11 @@ def test_encode_chunk_with_unicode(): df = pandas.DataFrame( numpy.random.randn(6, 4), index=range(6), columns=list("ABCD") ) - df["s"] = u"信用卡" + df["s"] = "信用卡" csv_buffer = load.encode_chunk(df) csv_bytes = csv_buffer.read() csv_string = csv_bytes.decode("utf-8") - assert u"信用卡" in csv_string + assert "信用卡" in csv_string def test_encode_chunk_with_floats(): @@ -63,7 +63,9 @@ def test_encode_chunk_with_floats(): csv_buffer = load.encode_chunk(input_df) round_trip = pandas.read_csv(csv_buffer, header=None, float_precision="round_trip") pandas.testing.assert_frame_equal( - round_trip, input_df, check_exact=True, + round_trip, + input_df, + check_exact=True, ) @@ -312,7 +314,8 @@ def test_cast_dataframe_for_parquet_w_string_date(): { "row_num": [0, 1, 2], "date_col": pandas.Series( - ["2021-04-17", "1999-12-31", "2038-01-19"], dtype="object", + ["2021-04-17", "1999-12-31", "2038-01-19"], + dtype="object", ), "row_num_2": [0, 1, 2], }, @@ -330,7 +333,8 @@ def test_cast_dataframe_for_parquet_w_string_date(): { "row_num": [0, 1, 2], "date_col": pandas.Series( - ["2021-04-17", "1999-12-31", "2038-01-19"], dtype=db_dtypes.DateDtype(), + ["2021-04-17", "1999-12-31", "2038-01-19"], + dtype=db_dtypes.DateDtype(), ), "row_num_2": [0, 1, 2], }, diff --git a/packages/pandas-gbq/tests/unit/test_schema.py b/packages/pandas-gbq/tests/unit/test_schema.py index d31ac2e980fb..7fdc616cdec9 100644 --- a/packages/pandas-gbq/tests/unit/test_schema.py +++ b/packages/pandas-gbq/tests/unit/test_schema.py @@ -30,8 +30,14 @@ def module_under_test(): ), # Original schema from API may contain legacy SQL datatype names. # https://github.com/pydata/pandas-gbq/issues/322 - ([{"name": "A", "type": "INTEGER"}], [{"name": "A", "type": "INT64"}],), - ([{"name": "A", "type": "BOOL"}], [{"name": "A", "type": "BOOLEAN"}],), + ( + [{"name": "A", "type": "INTEGER"}], + [{"name": "A", "type": "INT64"}], + ), + ( + [{"name": "A", "type": "BOOL"}], + [{"name": "A", "type": "BOOLEAN"}], + ), ( # TODO: include sub-fields when struct uploads are supported. [{"name": "A", "type": "STRUCT"}], @@ -79,7 +85,7 @@ def test_schema_is_subset_fails_if_not_subset(module_under_test): {"fields": [{"name": "col1", "type": "FLOAT"}]}, ), ( - pandas.DataFrame(data={"col1": [u"hello", u"world"]}), + pandas.DataFrame(data={"col1": ["hello", "world"]}), {"fields": [{"name": "col1", "type": "STRING"}]}, ), ( @@ -90,7 +96,7 @@ def test_schema_is_subset_fails_if_not_subset(module_under_test): pandas.DataFrame( data={ "col1": [datetime.datetime.now()], - "col2": [u"hello"], + "col2": ["hello"], "col3": [3.14], "col4": [True], "col5": [4], diff --git a/packages/pandas-gbq/tests/unit/test_to_gbq.py b/packages/pandas-gbq/tests/unit/test_to_gbq.py index a2fa800cc728..c8b419edd111 100644 --- a/packages/pandas-gbq/tests/unit/test_to_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_to_gbq.py @@ -41,8 +41,8 @@ def test_to_gbq_create_dataset_translates_exception(mock_bigquery_client): mock_bigquery_client.get_dataset.side_effect = google.api_core.exceptions.NotFound( "my_dataset" ) - mock_bigquery_client.create_dataset.side_effect = google.api_core.exceptions.InternalServerError( - "something went wrong" + mock_bigquery_client.create_dataset.side_effect = ( + google.api_core.exceptions.InternalServerError("something went wrong") ) with pytest.raises(gbq.GenericGBQException): From 97429b7a76237e6f25e6d81f701b7b0edc17db4a Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 30 Mar 2022 15:22:39 -0500 Subject: [PATCH 287/519] doc: show conventional commits instructions (#504) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - [ ] Make sure to open an issue as a [bug/issue](https://github.com/googleapis/python-bigquery-pandas/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea - [ ] Ensure the tests and linter pass - [ ] Code coverage does not decrease (if any source code was changed) - [ ] Appropriate docs were updated (if necessary) Towards #367 🦕 --- packages/pandas-gbq/docs/contributing.rst | 45 ++++++++++++----------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/packages/pandas-gbq/docs/contributing.rst b/packages/pandas-gbq/docs/contributing.rst index 4a17e8033b89..891a1799edc2 100644 --- a/packages/pandas-gbq/docs/contributing.rst +++ b/packages/pandas-gbq/docs/contributing.rst @@ -14,7 +14,7 @@ All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. If you are simply looking to start working with the *pandas-gbq* codebase, navigate to the -`GitHub "issues" tab `_ and start looking through +`GitHub "issues" tab `_ and start looking through interesting issues. Or maybe through using *pandas-gbq* you have an idea of your own or are looking for something @@ -71,7 +71,7 @@ It can very quickly become overwhelming, but sticking to the guidelines below wi straightforward and mostly trouble free. As always, if you are having difficulties please feel free to ask for help. -The code is hosted on `GitHub `_. To +The code is hosted on `GitHub `_. To contribute you will need to sign up for a `free GitHub account `_. We use `Git `_ for version control to allow many people to work together on the project. @@ -95,12 +95,12 @@ Forking ------- You will need your own fork to work on the code. Go to the `pandas-gbq project -page `_ and hit the ``Fork`` button. You will +page `_ and hit the ``Fork`` button. You will want to clone your fork to your machine:: git clone git@github.com:your-user-name/pandas-gbq.git pandas-gbq-yourname cd pandas-gbq-yourname - git remote add upstream git://github.com/pydata/pandas-gbq.git + git remote add upstream git://github.com/googleapis/python-bigquery-pandas.git This creates the directory `pandas-gbq-yourname` and connects your repository to the upstream (main project) *pandas-gbq* repository. @@ -153,10 +153,10 @@ Create a new conda environment and install the necessary dependencies .. code-block:: shell $ conda create -n my-env --channel conda-forge \ + db-dtypes \ pandas \ - google-auth-oauthlib \ - google-api-python-client \ - google-auth-httplib2 + pydata-google-auth \ + google-cloud-bigquery $ source activate my-env Install pandas-gbq in development mode @@ -307,7 +307,7 @@ Integration tests are skipped in pull requests because the credentials that are required for running Google BigQuery integration tests are `configured in the CircleCI web interface `_ -and are only accessible from the pydata/pandas-gbq repository. The +and are only accessible from the googleapis/python-bigquery-pandas repository. The credentials won't be available on forks of pandas-gbq. Here are the steps to run gbq integration tests on a forked repository: @@ -327,8 +327,8 @@ run gbq integration tests on a forked repository: sensitive data and you do not want their contents being exposed in build logs. #. Your branch should be tested automatically once it is pushed. You can check - the status by visiting your Travis branches page which exists at the - following location: https://circleci.com/gh/your-username/pandas-gbq . + the status by visiting your Circle CI branches page which exists at the + following location: https://circleci.com/gh/your-username/python-bigquery-pandas. Click on a build job for your branch. Documenting your code @@ -374,16 +374,17 @@ Doing 'git status' again should give something like:: # Finally, commit your changes to your local repository with an explanatory message. *pandas-gbq* -uses a convention for commit message prefixes and layout. Here are -some common prefixes along with general guidelines for when to use them: - - * ENH: Enhancement, new functionality - * BUG: Bug fix - * DOC: Additions/updates to documentation - * TST: Additions/updates to tests - * BLD: Updates to the build process/scripts - * PERF: Performance improvement - * CLN: Code cleanup +uses `conventional commit message prefixes +`_ and layout. Here are some +common prefixes along with general guidelines for when to use them: + + * feat: Enhancement, new functionality + * fix: Bug fix, performance improvement + * doc: Additions/updates to documentation + * deps: Change to package dependencies + * test: Additions/updates to tests + * chore: Updates to the build process/scripts + * refactor: Code cleanup The following defines how a commit message should be structured. Please reference the relevant GitHub issues in your commit message using GH1234 or #1234. Either style @@ -441,8 +442,8 @@ like:: origin git@github.com:yourname/pandas-gbq.git (fetch) origin git@github.com:yourname/pandas-gbq.git (push) - upstream git://github.com/pydata/pandas-gbq.git (fetch) - upstream git://github.com/pydata/pandas-gbq.git (push) + upstream git://github.com/googleapis/python-bigquery-pandas.git (fetch) + upstream git://github.com/googleapis/python-bigquery-pandas.git (push) Now your code is on GitHub, but it is not yet a part of the *pandas-gbq* project. For that to happen, a pull request needs to be submitted on GitHub. From 071942bb36da55175eb628926938db4bccf9a6b5 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Wed, 30 Mar 2022 21:04:18 +0000 Subject: [PATCH 288/519] chore(python): add E231 to .flake8 ignore list (#508) Source-Link: https://github.com/googleapis/synthtool/commit/7ff4aad2ec5af0380e8bd6da1fa06eaadf24ec81 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:462782b0b492346b2d9099aaff52206dd30bc8e031ea97082e6facecc2373244 --- packages/pandas-gbq/.flake8 | 2 +- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/.flake8 b/packages/pandas-gbq/.flake8 index 29227d4cf419..2e438749863d 100644 --- a/packages/pandas-gbq/.flake8 +++ b/packages/pandas-gbq/.flake8 @@ -16,7 +16,7 @@ # Generated by synthtool. DO NOT EDIT! [flake8] -ignore = E203, E266, E501, W503 +ignore = E203, E231, E266, E501, W503 exclude = # Exclude generated code. **/proto/** diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 87dd00611576..9e0a9356b6eb 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:7cffbc10910c3ab1b852c05114a08d374c195a81cdec1d4a67a1d129331d0bfe + digest: sha256:462782b0b492346b2d9099aaff52206dd30bc8e031ea97082e6facecc2373244 From e30a6f917dc70bf88415380937164b7c2b361e8d Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Thu, 31 Mar 2022 01:51:38 +0200 Subject: [PATCH 289/519] chore(deps): update all dependencies (#507) * chore(deps): update all dependencies * revert upgrade for environment specific pin; use `===` to prevent future updates * update environment pin Co-authored-by: Anthonios Partheniou --- packages/pandas-gbq/samples/snippets/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 0cfd8f264919..4cfef0a63fd4 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,5 +1,5 @@ google-cloud-bigquery-storage==2.13.0 -google-cloud-bigquery==2.34.2 -pandas==1.3.5; python_version < '3.8' +google-cloud-bigquery==3.0.1 +pandas===1.3.5; python_version == '3.7' pandas==1.4.1; python_version >= '3.8' pyarrow==7.0.0 From 5b77feb52d79dde401c01cb3ae62c8594b8ee4c3 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 1 Apr 2022 00:14:20 +0000 Subject: [PATCH 290/519] chore(python): update .pre-commit-config.yaml to use black==22.3.0 (#509) Source-Link: https://github.com/googleapis/synthtool/commit/7804ade3daae0d66649bee8df6c55484c6580b8d Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:eede5672562a32821444a8e803fb984a6f61f2237ea3de229d2de24453f4ae7d --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 3 ++- packages/pandas-gbq/.pre-commit-config.yaml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 9e0a9356b6eb..22cc254afa2c 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,4 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:462782b0b492346b2d9099aaff52206dd30bc8e031ea97082e6facecc2373244 + digest: sha256:eede5672562a32821444a8e803fb984a6f61f2237ea3de229d2de24453f4ae7d +# created: 2022-03-30T23:44:26.560599165Z diff --git a/packages/pandas-gbq/.pre-commit-config.yaml b/packages/pandas-gbq/.pre-commit-config.yaml index 62eb5a77d9a3..46d237160f6d 100644 --- a/packages/pandas-gbq/.pre-commit-config.yaml +++ b/packages/pandas-gbq/.pre-commit-config.yaml @@ -22,7 +22,7 @@ repos: - id: end-of-file-fixer - id: check-yaml - repo: https://github.com/psf/black - rev: 19.10b0 + rev: 22.3.0 hooks: - id: black - repo: https://gitlab.com/pycqa/flake8 From b1a3f7103f5dc177c90f3ea8d28aefcc47522ea1 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 1 Apr 2022 02:24:17 +0000 Subject: [PATCH 291/519] chore(python): Enable size-label bot (#510) Source-Link: https://github.com/googleapis/synthtool/commit/06e82790dd719a165ad32b8a06f8f6ec3e3cae0f Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:b3500c053313dc34e07b1632ba9e4e589f4f77036a7cf39e1fe8906811ae0fce --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 ++-- packages/pandas-gbq/.github/auto-label.yaml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 packages/pandas-gbq/.github/auto-label.yaml diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 22cc254afa2c..58a0b153bf0e 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:eede5672562a32821444a8e803fb984a6f61f2237ea3de229d2de24453f4ae7d -# created: 2022-03-30T23:44:26.560599165Z + digest: sha256:b3500c053313dc34e07b1632ba9e4e589f4f77036a7cf39e1fe8906811ae0fce +# created: 2022-04-01T01:42:03.609279246Z diff --git a/packages/pandas-gbq/.github/auto-label.yaml b/packages/pandas-gbq/.github/auto-label.yaml new file mode 100644 index 000000000000..09c8d735b456 --- /dev/null +++ b/packages/pandas-gbq/.github/auto-label.yaml @@ -0,0 +1,2 @@ +requestsize: + enabled: true From 8360e2445b66a30a17d69acf7efd2d094830c5d7 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 1 Apr 2022 19:42:26 +0000 Subject: [PATCH 292/519] chore(python): refactor unit / system test dependency install (#512) Source-Link: https://github.com/googleapis/synthtool/commit/993985f0fc4b37152e588f0549bcbdaf34666023 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:1894490910e891a385484514b22eb5133578897eb5b3c380e6d8ad475c6647cd --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 +- packages/pandas-gbq/noxfile.py | 118 ++++++++++++++---- 2 files changed, 95 insertions(+), 27 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 58a0b153bf0e..fa5762290c5b 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:b3500c053313dc34e07b1632ba9e4e589f4f77036a7cf39e1fe8906811ae0fce -# created: 2022-04-01T01:42:03.609279246Z + digest: sha256:1894490910e891a385484514b22eb5133578897eb5b3c380e6d8ad475c6647cd +# created: 2022-04-01T15:48:07.524222836Z diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 11bb1c90b75c..2b91aa41e717 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -21,16 +21,48 @@ import pathlib import re import shutil +import warnings import nox - BLACK_VERSION = "black==22.3.0" BLACK_PATHS = ["docs", "pandas_gbq", "tests", "noxfile.py", "setup.py"] DEFAULT_PYTHON_VERSION = "3.8" -SYSTEM_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] + UNIT_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] +UNIT_TEST_STANDARD_DEPENDENCIES = [ + "mock", + "asyncmock", + "pytest", + "pytest-cov", + "pytest-asyncio", +] +UNIT_TEST_EXTERNAL_DEPENDENCIES = [ + "freezegun", +] +UNIT_TEST_LOCAL_DEPENDENCIES = [] +UNIT_TEST_DEPENDENCIES = [] +UNIT_TEST_EXTRAS = [ + "tqdm", +] +UNIT_TEST_EXTRAS_BY_PYTHON = { + "3.9": [], +} + +SYSTEM_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] +SYSTEM_TEST_STANDARD_DEPENDENCIES = [ + "mock", + "pytest", + "google-cloud-testutils", +] +SYSTEM_TEST_EXTERNAL_DEPENDENCIES = [] +SYSTEM_TEST_LOCAL_DEPENDENCIES = [] +SYSTEM_TEST_DEPENDENCIES = [] +SYSTEM_TEST_EXTRAS = [ + "tqdm", +] +SYSTEM_TEST_EXTRAS_BY_PYTHON = {} CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() @@ -82,28 +114,41 @@ def lint_setup_py(session): session.run("python", "setup.py", "check", "--restructuredtext", "--strict") +def install_unittest_dependencies(session, *constraints): + standard_deps = UNIT_TEST_STANDARD_DEPENDENCIES + UNIT_TEST_DEPENDENCIES + session.install(*standard_deps, *constraints) + + if UNIT_TEST_EXTERNAL_DEPENDENCIES: + warnings.warn( + "'unit_test_external_dependencies' is deprecated. Instead, please " + "use 'unit_test_dependencies' or 'unit_test_local_dependencies'.", + DeprecationWarning, + ) + session.install(*UNIT_TEST_EXTERNAL_DEPENDENCIES, *constraints) + + if UNIT_TEST_LOCAL_DEPENDENCIES: + session.install(*UNIT_TEST_LOCAL_DEPENDENCIES, *constraints) + + if UNIT_TEST_EXTRAS_BY_PYTHON: + extras = UNIT_TEST_EXTRAS_BY_PYTHON.get(session.python, []) + elif UNIT_TEST_EXTRAS: + extras = UNIT_TEST_EXTRAS + else: + extras = [] + + if extras: + session.install("-e", f".[{','.join(extras)}]", *constraints) + else: + session.install("-e", ".", *constraints) + + def default(session): # Install all test dependencies, then install this package in-place. constraints_path = str( CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" ) - session.install( - "mock", - "asyncmock", - "pytest", - "pytest-cov", - "pytest-asyncio", - "-c", - constraints_path, - ) - session.install("freezegun", "-c", constraints_path) - - if session.python == "3.9": - extras = "" - else: - extras = "[tqdm]" - session.install("-e", f".{extras}", "-c", constraints_path) + install_unittest_dependencies(session, "-c", constraints_path) # Run py.test against the unit tests. session.run( @@ -127,6 +172,35 @@ def unit(session): default(session) +def install_systemtest_dependencies(session, *constraints): + + # Use pre-release gRPC for system tests. + session.install("--pre", "grpcio") + + session.install(*SYSTEM_TEST_STANDARD_DEPENDENCIES, *constraints) + + if SYSTEM_TEST_EXTERNAL_DEPENDENCIES: + session.install(*SYSTEM_TEST_EXTERNAL_DEPENDENCIES, *constraints) + + if SYSTEM_TEST_LOCAL_DEPENDENCIES: + session.install("-e", *SYSTEM_TEST_LOCAL_DEPENDENCIES, *constraints) + + if SYSTEM_TEST_DEPENDENCIES: + session.install("-e", *SYSTEM_TEST_DEPENDENCIES, *constraints) + + if SYSTEM_TEST_EXTRAS_BY_PYTHON: + extras = SYSTEM_TEST_EXTRAS_BY_PYTHON.get(session.python, []) + elif SYSTEM_TEST_EXTRAS: + extras = SYSTEM_TEST_EXTRAS + else: + extras = [] + + if extras: + session.install("-e", f".[{','.join(extras)}]", *constraints) + else: + session.install("-e", ".", *constraints) + + @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) def system(session): """Run the system test suite.""" @@ -149,13 +223,7 @@ def system(session): if not system_test_exists and not system_test_folder_exists: session.skip("System tests were not found") - # Use pre-release gRPC for system tests. - session.install("--pre", "grpcio") - - # Install all test dependencies, then install this package into the - # virtualenv's dist-packages. - session.install("mock", "pytest", "google-cloud-testutils", "-c", constraints_path) - session.install("-e", ".[tqdm]", "-c", constraints_path) + install_systemtest_dependencies(session, "-c", constraints_path) # Run py.test against the system tests. if system_test_exists: From 3e6202ede7c37ab4cddf2cc2480fbcb9b3376a4a Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Sun, 3 Apr 2022 21:44:06 +0200 Subject: [PATCH 293/519] chore(deps): update dependency pandas to v1.4.2 (#513) --- packages/pandas-gbq/samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 4cfef0a63fd4..52078d440190 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,5 +1,5 @@ google-cloud-bigquery-storage==2.13.0 google-cloud-bigquery==3.0.1 pandas===1.3.5; python_version == '3.7' -pandas==1.4.1; python_version >= '3.8' +pandas==1.4.2; python_version >= '3.8' pyarrow==7.0.0 From 1bad5e345433a9ffcae32f89e62f5c5b5ce63d2f Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 4 Apr 2022 10:38:31 -0500 Subject: [PATCH 294/519] chore: add ROADMAP document describing the purpose of the package (#505) * doc: add ROADMAP document describing the purpose of the package * additional thoughts Co-authored-by: Lo Ferris <50979514+loferris@users.noreply.github.com> Co-authored-by: Anthonios Partheniou --- packages/pandas-gbq/ROADMAP.md | 57 ++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 packages/pandas-gbq/ROADMAP.md diff --git a/packages/pandas-gbq/ROADMAP.md b/packages/pandas-gbq/ROADMAP.md new file mode 100644 index 000000000000..d4053cfa5279 --- /dev/null +++ b/packages/pandas-gbq/ROADMAP.md @@ -0,0 +1,57 @@ +# pandas-gbq Roadmap + +The purpose of this package is to provide a small subset of BigQuery +functionality that maps well to +[pandas.read_gbq](https://pandas.pydata.org/docs/reference/api/pandas.read_gbq.html#pandas.read_gbq) +and +[pandas.DataFrame.to_gbq](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_gbq.html#pandas.DataFrame.to_gbq). +Those methods in the pandas library are a thin wrapper to the equivalent +methods in this package. + +## Adding features to pandas-gbq + +Considerations when adding new features to pandas-gbq: + +* New method? Consider an alternative, as the core focus of this library is + `read_gbq` and `to_gbq`. +* Breaking change to an existing parameter? Consider an alternative, as folks + could be using an older version of `pandas` that doesn't account for the + change when a newer version of `pandas-gbq` is installed. If you must, please + follow a 1+ year deprecation timeline. +* New parameter? Go for it! Be sure to also send a PR to `pandas` after the + feature is released so that folks using the `pandas` wrapper can take + advantage of it. +* New data type? OK. If there's not a good mapping to an existing `pandas` + dtype, consider adding one to the `db-dtypes` package. + +## Vision + +The `pandas-gbq` package should do the "right thing" by default. This means you +should carefully choose dtypes for maximum compatibility with BigQuery and +avoid data loss. As new data types are added to BigQuery that don't have good +equivalents yet in the `pandas` ecosystem, equivalent dtypes should be added to +the `db-dtypes` package. + +As new features are added that might improve performance, `pandas-gbq` should +offer easy ways to use them without sacrificing usability. For example, one +might consider using the `api_method` parameter of `to_gbq` to support the +BigQuery Storage Write API. + +A note on `pandas.read_sql`: we'd like to be compatible with this too, for folks +that need better performance compared to the SQLAlchemy connector. + +## Usability + +Unlike the more object-oriented client-libraries, it's natural to have a method +with many parameters in the Python data science ecosystem. That said, the +`configuration` argument is provided, which takes the REST representation of +the job configuration so that power users can use new features without the need +for an explicit parameter being added. + +## Conclusion + +Keep it simple. + +Don't break existing users. + +Do the right thing by default. From 801dc0e5fd07e7611581fe0150db682ffda3a33a Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Wed, 6 Apr 2022 13:02:30 +0200 Subject: [PATCH 295/519] chore(deps): update dependency google-cloud-bigquery-storage to v2.13.1 (#514) --- packages/pandas-gbq/samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 52078d440190..ef2c64d014bb 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,4 +1,4 @@ -google-cloud-bigquery-storage==2.13.0 +google-cloud-bigquery-storage==2.13.1 google-cloud-bigquery==3.0.1 pandas===1.3.5; python_version == '3.7' pandas==1.4.2; python_version >= '3.8' From 9581542cda53c1816c76dd61d83bbd27db15a19c Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Wed, 6 Apr 2022 12:08:35 +0000 Subject: [PATCH 296/519] chore(python): add license header to auto-label.yaml (#515) Source-Link: https://github.com/googleapis/synthtool/commit/eb78c980b52c7c6746d2edb77d9cf7aaa99a2aab Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:8a5d3f6a2e43ed8293f34e06a2f56931d1e88a2694c3bb11b15df4eb256ad163 --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 ++-- packages/pandas-gbq/.github/auto-label.yaml | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index fa5762290c5b..bc893c979e20 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:1894490910e891a385484514b22eb5133578897eb5b3c380e6d8ad475c6647cd -# created: 2022-04-01T15:48:07.524222836Z + digest: sha256:8a5d3f6a2e43ed8293f34e06a2f56931d1e88a2694c3bb11b15df4eb256ad163 +# created: 2022-04-06T10:30:21.687684602Z diff --git a/packages/pandas-gbq/.github/auto-label.yaml b/packages/pandas-gbq/.github/auto-label.yaml index 09c8d735b456..41bff0b5375a 100644 --- a/packages/pandas-gbq/.github/auto-label.yaml +++ b/packages/pandas-gbq/.github/auto-label.yaml @@ -1,2 +1,15 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. requestsize: enabled: true From b65d2255dc5223cd7b7a9aafc08aa0e4c6f90dc4 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 21 Apr 2022 15:41:57 -0400 Subject: [PATCH 297/519] chore(python): add nox session to sort python imports (#518) Source-Link: https://github.com/googleapis/synthtool/commit/1b71c10e20de7ed3f97f692f99a0e3399b67049f Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:00c9d764fd1cd56265f12a5ef4b99a0c9e87cf261018099141e2ca5158890416 Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 +-- packages/pandas-gbq/noxfile.py | 27 ++++++++++++++++--- .../pandas-gbq/samples/snippets/noxfile.py | 22 +++++++++++++++ 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index bc893c979e20..7c454abf76f3 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:8a5d3f6a2e43ed8293f34e06a2f56931d1e88a2694c3bb11b15df4eb256ad163 -# created: 2022-04-06T10:30:21.687684602Z + digest: sha256:00c9d764fd1cd56265f12a5ef4b99a0c9e87cf261018099141e2ca5158890416 +# created: 2022-04-20T23:42:53.970438194Z diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 2b91aa41e717..28db2ecf1c77 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -26,7 +26,8 @@ import nox BLACK_VERSION = "black==22.3.0" -BLACK_PATHS = ["docs", "pandas_gbq", "tests", "noxfile.py", "setup.py"] +ISORT_VERSION = "isort==5.10.1" +LINT_PATHS = ["docs", "pandas_gbq", "tests", "noxfile.py", "setup.py"] DEFAULT_PYTHON_VERSION = "3.8" @@ -92,7 +93,7 @@ def lint(session): session.run( "black", "--check", - *BLACK_PATHS, + *LINT_PATHS, ) session.run("flake8", "pandas_gbq", "tests") @@ -103,7 +104,27 @@ def blacken(session): session.install(BLACK_VERSION) session.run( "black", - *BLACK_PATHS, + *LINT_PATHS, + ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def format(session): + """ + Run isort to sort imports. Then run black + to format code to uniform standard. + """ + session.install(BLACK_VERSION, ISORT_VERSION) + # Use the --fss option to sort imports using strict alphabetical order. + # See https://pycqa.github.io/isort/docs/configuration/options.html#force-sort-within-sections + session.run( + "isort", + "--fss", + *LINT_PATHS, + ) + session.run( + "black", + *LINT_PATHS, ) diff --git a/packages/pandas-gbq/samples/snippets/noxfile.py b/packages/pandas-gbq/samples/snippets/noxfile.py index 25f87a215d4c..a40410b56369 100644 --- a/packages/pandas-gbq/samples/snippets/noxfile.py +++ b/packages/pandas-gbq/samples/snippets/noxfile.py @@ -30,6 +30,7 @@ # WARNING - WARNING - WARNING - WARNING - WARNING BLACK_VERSION = "black==22.3.0" +ISORT_VERSION = "isort==5.10.1" # Copy `noxfile_config.py` to your directory and modify it instead. @@ -168,12 +169,33 @@ def lint(session: nox.sessions.Session) -> None: @nox.session def blacken(session: nox.sessions.Session) -> None: + """Run black. Format code to uniform standard.""" session.install(BLACK_VERSION) python_files = [path for path in os.listdir(".") if path.endswith(".py")] session.run("black", *python_files) +# +# format = isort + black +# + + +@nox.session +def format(session: nox.sessions.Session) -> None: + """ + Run isort to sort imports. Then run black + to format code to uniform standard. + """ + session.install(BLACK_VERSION, ISORT_VERSION) + python_files = [path for path in os.listdir(".") if path.endswith(".py")] + + # Use the --fss option to sort imports using strict alphabetical order. + # See https://pycqa.github.io/isort/docs/configuration/options.html#force-sort-within-sections + session.run("isort", "--fss", *python_files) + session.run("black", *python_files) + + # # Sample Tests # From 68246f64e2cc1bfad1ea72958272d034ae6244c8 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 21 Apr 2022 16:29:11 -0400 Subject: [PATCH 298/519] chore(python): use ubuntu 22.04 in docs image (#520) Source-Link: https://github.com/googleapis/synthtool/commit/f15cc72fb401b4861cedebb10af74afe428fb1f8 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:bc5eed3804aec2f05fad42aacf973821d9500c174015341f721a984a0825b6fd Co-authored-by: Owl Bot Co-authored-by: Anthonios Partheniou --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 ++-- .../pandas-gbq/.kokoro/docker/docs/Dockerfile | 20 +++++++++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 7c454abf76f3..64f82d6bf4bc 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:00c9d764fd1cd56265f12a5ef4b99a0c9e87cf261018099141e2ca5158890416 -# created: 2022-04-20T23:42:53.970438194Z + digest: sha256:bc5eed3804aec2f05fad42aacf973821d9500c174015341f721a984a0825b6fd +# created: 2022-04-21T15:43:16.246106921Z diff --git a/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile b/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile index 4e1b1fb8b5a5..238b87b9d1c9 100644 --- a/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile +++ b/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ubuntu:20.04 +from ubuntu:22.04 ENV DEBIAN_FRONTEND noninteractive @@ -60,8 +60,24 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ && rm -f /var/cache/apt/archives/*.deb +###################### Install python 3.8.11 + +# Download python 3.8.11 +RUN wget https://www.python.org/ftp/python/3.8.11/Python-3.8.11.tgz + +# Extract files +RUN tar -xvf Python-3.8.11.tgz + +# Install python 3.8.11 +RUN ./Python-3.8.11/configure --enable-optimizations +RUN make altinstall + +###################### Install pip RUN wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \ - && python3.8 /tmp/get-pip.py \ + && python3 /tmp/get-pip.py \ && rm /tmp/get-pip.py +# Test pip +RUN python3 -m pip + CMD ["python3.8"] From 6c377ac1827a7b0846fdd2f6f806c4c8ddae3064 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Mon, 25 Apr 2022 18:04:14 +0200 Subject: [PATCH 299/519] chore(deps): update dependency pytest to v7.1.2 (#521) --- packages/pandas-gbq/samples/snippets/requirements-test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements-test.txt b/packages/pandas-gbq/samples/snippets/requirements-test.txt index e227954a1e25..0b8f81a49d17 100644 --- a/packages/pandas-gbq/samples/snippets/requirements-test.txt +++ b/packages/pandas-gbq/samples/snippets/requirements-test.txt @@ -1,2 +1,2 @@ google-cloud-testutils==1.3.1 -pytest==7.1.1 +pytest==7.1.2 From 7301d66ea5265e54eae879a6aab24e48d0a5ce17 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 5 May 2022 13:19:56 -0400 Subject: [PATCH 300/519] chore: [autoapprove] update readme_gen.py to include autoescape True (#522) Source-Link: https://github.com/googleapis/synthtool/commit/6b4d5a6407d740beb4158b302194a62a4108a8a6 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:f792ee1320e03eda2d13a5281a2989f7ed8a9e50b73ef6da97fac7e1e850b149 Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 ++-- packages/pandas-gbq/scripts/readme-gen/readme_gen.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 64f82d6bf4bc..b631901e99f4 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:bc5eed3804aec2f05fad42aacf973821d9500c174015341f721a984a0825b6fd -# created: 2022-04-21T15:43:16.246106921Z + digest: sha256:f792ee1320e03eda2d13a5281a2989f7ed8a9e50b73ef6da97fac7e1e850b149 +# created: 2022-05-05T15:17:27.599381182Z diff --git a/packages/pandas-gbq/scripts/readme-gen/readme_gen.py b/packages/pandas-gbq/scripts/readme-gen/readme_gen.py index d309d6e97518..91b59676bfc7 100644 --- a/packages/pandas-gbq/scripts/readme-gen/readme_gen.py +++ b/packages/pandas-gbq/scripts/readme-gen/readme_gen.py @@ -28,7 +28,10 @@ jinja_env = jinja2.Environment( trim_blocks=True, loader=jinja2.FileSystemLoader( - os.path.abspath(os.path.join(os.path.dirname(__file__), 'templates')))) + os.path.abspath(os.path.join(os.path.dirname(__file__), "templates")) + ), + autoescape=True, +) README_TMPL = jinja_env.get_template('README.tmpl.rst') From ba504342917514cf9ca495ae073dd878f4dc03a0 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 5 May 2022 19:57:44 -0400 Subject: [PATCH 301/519] chore(python): auto approve template changes (#524) Source-Link: https://github.com/googleapis/synthtool/commit/453a5d9c9a55d1969240a37d36cec626d20a9024 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:81ed5ecdfc7cac5b699ba4537376f3563f6f04122c4ec9e735d3b3dc1d43dd32 Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 ++-- packages/pandas-gbq/.github/auto-approve.yml | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 packages/pandas-gbq/.github/auto-approve.yml diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index b631901e99f4..757c9dca75ad 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:f792ee1320e03eda2d13a5281a2989f7ed8a9e50b73ef6da97fac7e1e850b149 -# created: 2022-05-05T15:17:27.599381182Z + digest: sha256:81ed5ecdfc7cac5b699ba4537376f3563f6f04122c4ec9e735d3b3dc1d43dd32 +# created: 2022-05-05T22:08:23.383410683Z diff --git a/packages/pandas-gbq/.github/auto-approve.yml b/packages/pandas-gbq/.github/auto-approve.yml new file mode 100644 index 000000000000..311ebbb853a9 --- /dev/null +++ b/packages/pandas-gbq/.github/auto-approve.yml @@ -0,0 +1,3 @@ +# https://github.com/googleapis/repo-automation-bots/tree/main/packages/auto-approve +processes: + - "OwlBotTemplateChanges" From 287d76c7f91bc6eff5feb7564a9fc297f23478c9 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Mon, 9 May 2022 22:55:35 +0200 Subject: [PATCH 302/519] fix(deps): allow pyarrow v8 (#525) * chore(deps): update dependency pyarrow to v8 * fix(deps): allow pyarrow v8 * chore(deps): update dependency google-cloud-bigquery to v3.1.0 Co-authored-by: Anthonios Partheniou --- packages/pandas-gbq/samples/snippets/requirements.txt | 4 ++-- packages/pandas-gbq/setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index ef2c64d014bb..6000d3442eef 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,5 +1,5 @@ google-cloud-bigquery-storage==2.13.1 -google-cloud-bigquery==3.0.1 +google-cloud-bigquery==3.1.0 pandas===1.3.5; python_version == '3.7' pandas==1.4.2; python_version >= '3.8' -pyarrow==7.0.0 +pyarrow==8.0.0 diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index d395e71f81da..c070782d59d6 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -26,7 +26,7 @@ "db-dtypes >=0.3.1,<2.0.0", "numpy >=1.16.6", "pandas >=0.24.2", - "pyarrow >=3.0.0, <8.0dev", + "pyarrow >=3.0.0, <9.0dev", "pydata-google-auth", # Note: google-api-core and google-auth are also included via transitive # dependency on google-cloud-bigquery, but this library also uses them From bd5f9f6ae5a1bf63b8424d0cc9ba8c11f538ca6c Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Thu, 19 May 2022 08:36:14 -0400 Subject: [PATCH 303/519] chore(main): release 0.17.5 (#526) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- packages/pandas-gbq/CHANGELOG.md | 7 +++++++ packages/pandas-gbq/pandas_gbq/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index d94f687f33c7..35793c80a512 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +### [0.17.5](https://github.com/googleapis/python-bigquery-pandas/compare/v0.17.4...v0.17.5) (2022-05-09) + + +### Bug Fixes + +* **deps:** allow pyarrow v8 ([#525](https://github.com/googleapis/python-bigquery-pandas/issues/525)) ([a4ee0df](https://github.com/googleapis/python-bigquery-pandas/commit/a4ee0dffae0580a7509d5d6014edb46e05394717)) + ### [0.17.4](https://github.com/googleapis/python-bigquery-pandas/compare/v0.17.3...v0.17.4) (2022-03-14) diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py index 862c78c93c7f..17d9be75dde5 100644 --- a/packages/pandas-gbq/pandas_gbq/version.py +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -2,4 +2,4 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -__version__ = "0.17.4" +__version__ = "0.17.5" From 709140086c2f34d29647c6cb13949851f29de939 Mon Sep 17 00:00:00 2001 From: Dan Lee <71398022+dandhlee@users.noreply.github.com> Date: Thu, 2 Jun 2022 20:42:02 -0400 Subject: [PATCH 304/519] docs: fix changelog header to consistent size (#529) --- packages/pandas-gbq/CHANGELOG.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index 35793c80a512..2b9a16397396 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,13 +1,13 @@ # Changelog -### [0.17.5](https://github.com/googleapis/python-bigquery-pandas/compare/v0.17.4...v0.17.5) (2022-05-09) +## [0.17.5](https://github.com/googleapis/python-bigquery-pandas/compare/v0.17.4...v0.17.5) (2022-05-09) ### Bug Fixes * **deps:** allow pyarrow v8 ([#525](https://github.com/googleapis/python-bigquery-pandas/issues/525)) ([a4ee0df](https://github.com/googleapis/python-bigquery-pandas/commit/a4ee0dffae0580a7509d5d6014edb46e05394717)) -### [0.17.4](https://github.com/googleapis/python-bigquery-pandas/compare/v0.17.3...v0.17.4) (2022-03-14) +## [0.17.4](https://github.com/googleapis/python-bigquery-pandas/compare/v0.17.3...v0.17.4) (2022-03-14) ### Bug Fixes @@ -15,7 +15,7 @@ * avoid deprecated "out-of-band" authentication flow ([#500](https://github.com/googleapis/python-bigquery-pandas/issues/500)) ([4758e3a](https://github.com/googleapis/python-bigquery-pandas/commit/4758e3a9ccb82109aae65f76258b2910077e02dd)) * correctly transform query job timeout configuration and exceptions ([#492](https://github.com/googleapis/python-bigquery-pandas/issues/492)) ([d8c3900](https://github.com/googleapis/python-bigquery-pandas/commit/d8c3900eda5aa2cb5b663b2be569d639f6a028a9)) -### [0.17.3](https://github.com/googleapis/python-bigquery-pandas/compare/v0.17.2...v0.17.3) (2022-03-05) +## [0.17.3](https://github.com/googleapis/python-bigquery-pandas/compare/v0.17.2...v0.17.3) (2022-03-05) ### Bug Fixes @@ -24,14 +24,14 @@ * **deps:** require google-auth>=1.25.0 ([744a71c](https://github.com/googleapis/python-bigquery-pandas/commit/744a71c3d265d0e9a2ac25ca98dd0fa3ca68af6a)) * **deps:** require proto-plus>=1.15.0 ([744a71c](https://github.com/googleapis/python-bigquery-pandas/commit/744a71c3d265d0e9a2ac25ca98dd0fa3ca68af6a)) -### [0.17.2](https://github.com/googleapis/python-bigquery-pandas/compare/v0.17.1...v0.17.2) (2022-03-02) +## [0.17.2](https://github.com/googleapis/python-bigquery-pandas/compare/v0.17.1...v0.17.2) (2022-03-02) ### Dependencies * allow pyarrow 7.0 ([#487](https://github.com/googleapis/python-bigquery-pandas/issues/487)) ([39441b6](https://github.com/googleapis/python-bigquery-pandas/commit/39441b63fadd95810c535e7079d781e9eec72189)) -### [0.17.1](https://github.com/googleapis/python-bigquery-pandas/compare/v0.17.0...v0.17.1) (2022-02-24) +## [0.17.1](https://github.com/googleapis/python-bigquery-pandas/compare/v0.17.0...v0.17.1) (2022-02-24) ### Bug Fixes From 7ec04110c8c6109e76b5be0073a4de19aa220cda Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Thu, 9 Jun 2022 10:26:49 -0400 Subject: [PATCH 305/519] chore(main): release 0.17.6 (#530) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- packages/pandas-gbq/CHANGELOG.md | 7 +++++++ packages/pandas-gbq/pandas_gbq/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index 2b9a16397396..ad2ddcc84dc1 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.17.6](https://github.com/googleapis/python-bigquery-pandas/compare/v0.17.5...v0.17.6) (2022-06-03) + + +### Documentation + +* fix changelog header to consistent size ([#529](https://github.com/googleapis/python-bigquery-pandas/issues/529)) ([218e06a](https://github.com/googleapis/python-bigquery-pandas/commit/218e06af40e991f870649a8e958dfc1bc46f0ee8)) + ## [0.17.5](https://github.com/googleapis/python-bigquery-pandas/compare/v0.17.4...v0.17.5) (2022-05-09) diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py index 17d9be75dde5..716bb4b4862f 100644 --- a/packages/pandas-gbq/pandas_gbq/version.py +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -2,4 +2,4 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -__version__ = "0.17.5" +__version__ = "0.17.6" From ab48d912c8ac4cc6009e945ad15e599024d4ce1c Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Sun, 12 Jun 2022 11:06:32 -0400 Subject: [PATCH 306/519] chore: add prerelease nox session (#533) Source-Link: https://github.com/googleapis/synthtool/commit/050953d60f71b4ed4be563e032f03c192c50332f Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:65e656411895bff71cffcae97246966460160028f253c2e45b7a25d805a5b142 Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 +- .../.kokoro/continuous/prerelease-deps.cfg | 7 ++ .../.kokoro/presubmit/prerelease-deps.cfg | 7 ++ packages/pandas-gbq/noxfile.py | 64 +++++++++++++++++++ 4 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 packages/pandas-gbq/.kokoro/continuous/prerelease-deps.cfg create mode 100644 packages/pandas-gbq/.kokoro/presubmit/prerelease-deps.cfg diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 757c9dca75ad..2185b591844c 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:81ed5ecdfc7cac5b699ba4537376f3563f6f04122c4ec9e735d3b3dc1d43dd32 -# created: 2022-05-05T22:08:23.383410683Z + digest: sha256:65e656411895bff71cffcae97246966460160028f253c2e45b7a25d805a5b142 +# created: 2022-06-12T13:11:45.905884945Z diff --git a/packages/pandas-gbq/.kokoro/continuous/prerelease-deps.cfg b/packages/pandas-gbq/.kokoro/continuous/prerelease-deps.cfg new file mode 100644 index 000000000000..3595fb43f5c0 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/continuous/prerelease-deps.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "prerelease_deps" +} diff --git a/packages/pandas-gbq/.kokoro/presubmit/prerelease-deps.cfg b/packages/pandas-gbq/.kokoro/presubmit/prerelease-deps.cfg new file mode 100644 index 000000000000..3595fb43f5c0 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/presubmit/prerelease-deps.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "prerelease_deps" +} diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 28db2ecf1c77..37a32242ddc1 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -416,3 +416,67 @@ def docfx(session): os.path.join("docs", ""), os.path.join("docs", "_build", "html", ""), ) + + +@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) +def prerelease_deps(session): + """Run all tests with prerelease versions of dependencies installed.""" + + prerel_deps = [ + "protobuf", + "googleapis-common-protos", + "google-auth", + "grpcio", + "grpcio-status", + "google-api-core", + "proto-plus", + # dependencies of google-auth + "cryptography", + "pyasn1", + ] + + for dep in prerel_deps: + session.install("--pre", "--no-deps", "--upgrade", dep) + + # Remaining dependencies + other_deps = ["requests"] + session.install(*other_deps) + + session.install(*UNIT_TEST_STANDARD_DEPENDENCIES) + session.install(*SYSTEM_TEST_STANDARD_DEPENDENCIES) + + # Because we test minimum dependency versions on the minimum Python + # version, the first version we test with in the unit tests sessions has a + # constraints file containing all dependencies and extras. + with open( + CURRENT_DIRECTORY + / "testing" + / f"constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt", + encoding="utf-8", + ) as constraints_file: + constraints_text = constraints_file.read() + + # Ignore leading whitespace and comment lines. + deps = [ + match.group(1) + for match in re.finditer( + r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE + ) + ] + + # Don't overwrite prerelease packages. + deps = [dep for dep in deps if dep not in prerel_deps] + # We use --no-deps to ensure that pre-release versions aren't overwritten + # by the version ranges in setup.py. + session.install(*deps) + session.install("--no-deps", "-e", ".[all]") + + # Print out prerelease package versions + session.run( + "python", "-c", "import google.protobuf; print(google.protobuf.__version__)" + ) + session.run("python", "-c", "import grpc; print(grpc.__version__)") + + session.run("py.test", "tests/unit") + session.run("py.test", "tests/system") + session.run("py.test", "samples/snippets") From 609f7c8526025bda26f098fc85ec462356ba0754 Mon Sep 17 00:00:00 2001 From: Avishai Carmel Date: Thu, 16 Jun 2022 21:05:20 +0300 Subject: [PATCH 307/519] fix: allow `to_gbq` to run without `bigquery.tables.create` permission. (#539) --- packages/pandas-gbq/pandas_gbq/load.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/pandas-gbq/pandas_gbq/load.py b/packages/pandas-gbq/pandas_gbq/load.py index 2fcf26109bf0..1032806957bf 100644 --- a/packages/pandas-gbq/pandas_gbq/load.py +++ b/packages/pandas-gbq/pandas_gbq/load.py @@ -119,6 +119,7 @@ def load_parquet( ): job_config = bigquery.LoadJobConfig() job_config.write_disposition = "WRITE_APPEND" + job_config.create_disposition = "CREATE_NEVER" job_config.source_format = "PARQUET" if schema is not None: @@ -148,6 +149,7 @@ def load_csv( ): job_config = bigquery.LoadJobConfig() job_config.write_disposition = "WRITE_APPEND" + job_config.create_disposition = "CREATE_NEVER" job_config.source_format = "CSV" job_config.allow_quoted_newlines = True From fb395a8a0e711ed8b6e5bd63caac22e948f754fb Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Thu, 7 Jul 2022 23:40:34 +0200 Subject: [PATCH 308/519] chore(deps): update all dependencies (#531) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update all dependencies * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot Co-authored-by: Steffany Brown <30247553+steffnay@users.noreply.github.com> --- packages/pandas-gbq/samples/snippets/requirements-test.txt | 2 +- packages/pandas-gbq/samples/snippets/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements-test.txt b/packages/pandas-gbq/samples/snippets/requirements-test.txt index 0b8f81a49d17..46ad2b178732 100644 --- a/packages/pandas-gbq/samples/snippets/requirements-test.txt +++ b/packages/pandas-gbq/samples/snippets/requirements-test.txt @@ -1,2 +1,2 @@ -google-cloud-testutils==1.3.1 +google-cloud-testutils==1.3.2 pytest==7.1.2 diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 6000d3442eef..57a896327693 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,4 +1,4 @@ -google-cloud-bigquery-storage==2.13.1 +google-cloud-bigquery-storage==2.13.2 google-cloud-bigquery==3.1.0 pandas===1.3.5; python_version == '3.7' pandas==1.4.2; python_version >= '3.8' From d2462fc631f0cb95e904e31b5ebf1bd814bacd9d Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Mon, 11 Jul 2022 10:52:54 -0700 Subject: [PATCH 309/519] chore: update templated files (#544) * chore(python): drop python 3.6 Source-Link: https://github.com/googleapis/synthtool/commit/4f89b13af10d086458f9b379e56a614f9d6dab7b Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:e7bb19d47c13839fe8c147e50e02e8b6cf5da8edd1af8b82208cd6f66cc2829c * remove python 3.6 sample configs * exclude templated README * use python 3.8 for readthedocs build Co-authored-by: Owl Bot Co-authored-by: Anthonios Partheniou --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 +- .../.kokoro/samples/python3.6/common.cfg | 40 --------- .../.kokoro/samples/python3.6/continuous.cfg | 7 -- .../samples/python3.6/periodic-head.cfg | 11 --- .../.kokoro/samples/python3.6/periodic.cfg | 6 -- .../.kokoro/samples/python3.6/presubmit.cfg | 6 -- .../pandas-gbq/.kokoro/test-samples-impl.sh | 4 +- packages/pandas-gbq/.readthedocs.yml | 2 +- packages/pandas-gbq/noxfile.py | 83 ++++++++++++------- packages/pandas-gbq/owlbot.py | 1 + .../pandas-gbq/samples/snippets/noxfile.py | 2 +- .../templates/install_deps.tmpl.rst | 2 +- .../pandas-gbq/testing/constraints-3.7.txt | 1 + 13 files changed, 62 insertions(+), 107 deletions(-) delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.6/common.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.6/continuous.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.6/periodic-head.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.6/periodic.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.6/presubmit.cfg diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 2185b591844c..1ce608523524 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:65e656411895bff71cffcae97246966460160028f253c2e45b7a25d805a5b142 -# created: 2022-06-12T13:11:45.905884945Z + digest: sha256:e7bb19d47c13839fe8c147e50e02e8b6cf5da8edd1af8b82208cd6f66cc2829c +# created: 2022-07-05T18:31:20.838186805Z diff --git a/packages/pandas-gbq/.kokoro/samples/python3.6/common.cfg b/packages/pandas-gbq/.kokoro/samples/python3.6/common.cfg deleted file mode 100644 index bf7686db795a..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.6/common.cfg +++ /dev/null @@ -1,40 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Build logs will be here -action { - define_artifacts { - regex: "**/*sponge_log.xml" - } -} - -# Specify which tests to run -env_vars: { - key: "RUN_TESTS_SESSION" - value: "py-3.6" -} - -# Declare build specific Cloud project. -env_vars: { - key: "BUILD_SPECIFIC_GCLOUD_PROJECT" - value: "python-docs-samples-tests-py36" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-bigquery-pandas/.kokoro/test-samples.sh" -} - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" -} - -# Download secrets for samples -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" - -# Download trampoline resources. -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" - -# Use the trampoline script to run in docker. -build_file: "python-bigquery-pandas/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.6/continuous.cfg b/packages/pandas-gbq/.kokoro/samples/python3.6/continuous.cfg deleted file mode 100644 index 7218af1499e5..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.6/continuous.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} - diff --git a/packages/pandas-gbq/.kokoro/samples/python3.6/periodic-head.cfg b/packages/pandas-gbq/.kokoro/samples/python3.6/periodic-head.cfg deleted file mode 100644 index 98efde4dc99d..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.6/periodic-head.cfg +++ /dev/null @@ -1,11 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-bigquery-pandas/.kokoro/test-samples-against-head.sh" -} diff --git a/packages/pandas-gbq/.kokoro/samples/python3.6/periodic.cfg b/packages/pandas-gbq/.kokoro/samples/python3.6/periodic.cfg deleted file mode 100644 index 71cd1e597e38..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.6/periodic.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "False" -} diff --git a/packages/pandas-gbq/.kokoro/samples/python3.6/presubmit.cfg b/packages/pandas-gbq/.kokoro/samples/python3.6/presubmit.cfg deleted file mode 100644 index a1c8d9759c88..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.6/presubmit.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/test-samples-impl.sh b/packages/pandas-gbq/.kokoro/test-samples-impl.sh index 8a324c9c7bc6..2c6500cae0b9 100755 --- a/packages/pandas-gbq/.kokoro/test-samples-impl.sh +++ b/packages/pandas-gbq/.kokoro/test-samples-impl.sh @@ -33,7 +33,7 @@ export PYTHONUNBUFFERED=1 env | grep KOKORO # Install nox -python3.6 -m pip install --upgrade --quiet nox +python3.9 -m pip install --upgrade --quiet nox # Use secrets acessor service account to get secrets if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then @@ -76,7 +76,7 @@ for file in samples/**/requirements.txt; do echo "------------------------------------------------------------" # Use nox to execute the tests for the project. - python3.6 -m nox -s "$RUN_TESTS_SESSION" + python3.9 -m nox -s "$RUN_TESTS_SESSION" EXIT=$? # If this is a periodic build, send the test log to the FlakyBot. diff --git a/packages/pandas-gbq/.readthedocs.yml b/packages/pandas-gbq/.readthedocs.yml index 420befc5fcf3..9b3d685401ea 100644 --- a/packages/pandas-gbq/.readthedocs.yml +++ b/packages/pandas-gbq/.readthedocs.yml @@ -7,4 +7,4 @@ build: image: latest python: pip_install: true - version: 3.6 + version: 3.8 diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 37a32242ddc1..ae8869ff99aa 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -422,28 +422,15 @@ def docfx(session): def prerelease_deps(session): """Run all tests with prerelease versions of dependencies installed.""" - prerel_deps = [ - "protobuf", - "googleapis-common-protos", - "google-auth", - "grpcio", - "grpcio-status", - "google-api-core", - "proto-plus", - # dependencies of google-auth - "cryptography", - "pyasn1", - ] - - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--upgrade", dep) - - # Remaining dependencies - other_deps = ["requests"] - session.install(*other_deps) - + # Install all dependencies + session.install("-e", ".[all, tests, tracing]") session.install(*UNIT_TEST_STANDARD_DEPENDENCIES) - session.install(*SYSTEM_TEST_STANDARD_DEPENDENCIES) + system_deps_all = ( + SYSTEM_TEST_STANDARD_DEPENDENCIES + + SYSTEM_TEST_EXTERNAL_DEPENDENCIES + + SYSTEM_TEST_EXTRAS + ) + session.install(*system_deps_all) # Because we test minimum dependency versions on the minimum Python # version, the first version we test with in the unit tests sessions has a @@ -457,19 +444,44 @@ def prerelease_deps(session): constraints_text = constraints_file.read() # Ignore leading whitespace and comment lines. - deps = [ + constraints_deps = [ match.group(1) for match in re.finditer( r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE ) ] - # Don't overwrite prerelease packages. - deps = [dep for dep in deps if dep not in prerel_deps] - # We use --no-deps to ensure that pre-release versions aren't overwritten - # by the version ranges in setup.py. - session.install(*deps) - session.install("--no-deps", "-e", ".[all]") + session.install(*constraints_deps) + + if os.path.exists("samples/snippets/requirements.txt"): + session.install("-r", "samples/snippets/requirements.txt") + + if os.path.exists("samples/snippets/requirements-test.txt"): + session.install("-r", "samples/snippets/requirements-test.txt") + + prerel_deps = [ + "protobuf", + # dependency of grpc + "six", + "googleapis-common-protos", + "grpcio", + "grpcio-status", + "google-api-core", + "proto-plus", + "google-cloud-testutils", + # dependencies of google-cloud-testutils" + "click", + ] + + for dep in prerel_deps: + session.install("--pre", "--no-deps", "--upgrade", dep) + + # Remaining dependencies + other_deps = [ + "requests", + "google-auth", + ] + session.install(*other_deps) # Print out prerelease package versions session.run( @@ -478,5 +490,16 @@ def prerelease_deps(session): session.run("python", "-c", "import grpc; print(grpc.__version__)") session.run("py.test", "tests/unit") - session.run("py.test", "tests/system") - session.run("py.test", "samples/snippets") + + system_test_path = os.path.join("tests", "system.py") + system_test_folder_path = os.path.join("tests", "system") + + # Only run system tests if found. + if os.path.exists(system_test_path) or os.path.exists(system_test_folder_path): + session.run("py.test", "tests/system") + + snippets_test_path = os.path.join("samples", "snippets") + + # Only run samples tests if found. + if os.path.exists(snippets_test_path): + session.run("py.test", "samples/snippets") diff --git a/packages/pandas-gbq/owlbot.py b/packages/pandas-gbq/owlbot.py index 02eeb0693a5c..1ec8fbe39f81 100644 --- a/packages/pandas-gbq/owlbot.py +++ b/packages/pandas-gbq/owlbot.py @@ -55,6 +55,7 @@ # Mulit-processing note isn't relevant, as pandas_gbq is responsible for # creating clients, not the end user. "docs/multiprocessing.rst", + "README.rst", ], ) diff --git a/packages/pandas-gbq/samples/snippets/noxfile.py b/packages/pandas-gbq/samples/snippets/noxfile.py index a40410b56369..29b5bc852183 100644 --- a/packages/pandas-gbq/samples/snippets/noxfile.py +++ b/packages/pandas-gbq/samples/snippets/noxfile.py @@ -89,7 +89,7 @@ def get_pytest_env_vars() -> Dict[str, str]: # DO NOT EDIT - automatically generated. # All versions used to test samples. -ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10"] +ALL_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] # Any default versions that should be ignored. IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] diff --git a/packages/pandas-gbq/scripts/readme-gen/templates/install_deps.tmpl.rst b/packages/pandas-gbq/scripts/readme-gen/templates/install_deps.tmpl.rst index 275d649890d7..6f069c6c87a5 100644 --- a/packages/pandas-gbq/scripts/readme-gen/templates/install_deps.tmpl.rst +++ b/packages/pandas-gbq/scripts/readme-gen/templates/install_deps.tmpl.rst @@ -12,7 +12,7 @@ Install Dependencies .. _Python Development Environment Setup Guide: https://cloud.google.com/python/setup -#. Create a virtualenv. Samples are compatible with Python 3.6+. +#. Create a virtualenv. Samples are compatible with Python 3.7+. .. code-block:: bash diff --git a/packages/pandas-gbq/testing/constraints-3.7.txt b/packages/pandas-gbq/testing/constraints-3.7.txt index 2d4674eb6712..2d9b95f8ae2c 100644 --- a/packages/pandas-gbq/testing/constraints-3.7.txt +++ b/packages/pandas-gbq/testing/constraints-3.7.txt @@ -16,3 +16,4 @@ pandas==0.24.2 pyarrow==3.0.0 pydata-google-auth==0.1.2 tqdm==4.23.0 +protobuf==3.19.0 From 120b7101a03e0f9f41f626fc76c9c10e1a5e6399 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 11 Jul 2022 11:47:26 -0700 Subject: [PATCH 310/519] chore(main): release 0.17.7 (#540) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- packages/pandas-gbq/CHANGELOG.md | 7 +++++++ packages/pandas-gbq/pandas_gbq/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index ad2ddcc84dc1..d5073a0ab702 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.17.7](https://github.com/googleapis/python-bigquery-pandas/compare/v0.17.6...v0.17.7) (2022-07-11) + + +### Bug Fixes + +* allow `to_gbq` to run without `bigquery.tables.create` permission. ([#539](https://github.com/googleapis/python-bigquery-pandas/issues/539)) ([3988306](https://github.com/googleapis/python-bigquery-pandas/commit/3988306bd2cc7743d24e24d753730ba04462f018)) + ## [0.17.6](https://github.com/googleapis/python-bigquery-pandas/compare/v0.17.5...v0.17.6) (2022-06-03) diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py index 716bb4b4862f..63a5c240fac0 100644 --- a/packages/pandas-gbq/pandas_gbq/version.py +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -2,4 +2,4 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -__version__ = "0.17.6" +__version__ = "0.17.7" From be3c1f0695d5416608cc83df16a319d70627a7f9 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 29 Jul 2022 14:31:34 -0400 Subject: [PATCH 311/519] chore(python): fix prerelease session [autoapprove] (#546) Source-Link: https://github.com/googleapis/synthtool/commit/1b9ad7694e44ddb4d9844df55ff7af77b51a4435 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:9db98b055a7f8bd82351238ccaacfd3cda58cdf73012ab58b8da146368330021 Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 +-- packages/pandas-gbq/noxfile.py | 33 ++++++++++--------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 1ce608523524..0eb02fda4c09 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:e7bb19d47c13839fe8c147e50e02e8b6cf5da8edd1af8b82208cd6f66cc2829c -# created: 2022-07-05T18:31:20.838186805Z + digest: sha256:9db98b055a7f8bd82351238ccaacfd3cda58cdf73012ab58b8da146368330021 +# created: 2022-07-25T16:02:49.174178716Z diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index ae8869ff99aa..9ea981f4f0be 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -424,7 +424,8 @@ def prerelease_deps(session): # Install all dependencies session.install("-e", ".[all, tests, tracing]") - session.install(*UNIT_TEST_STANDARD_DEPENDENCIES) + unit_deps_all = UNIT_TEST_STANDARD_DEPENDENCIES + UNIT_TEST_EXTERNAL_DEPENDENCIES + session.install(*unit_deps_all) system_deps_all = ( SYSTEM_TEST_STANDARD_DEPENDENCIES + SYSTEM_TEST_EXTERNAL_DEPENDENCIES @@ -453,12 +454,6 @@ def prerelease_deps(session): session.install(*constraints_deps) - if os.path.exists("samples/snippets/requirements.txt"): - session.install("-r", "samples/snippets/requirements.txt") - - if os.path.exists("samples/snippets/requirements-test.txt"): - session.install("-r", "samples/snippets/requirements-test.txt") - prerel_deps = [ "protobuf", # dependency of grpc @@ -495,11 +490,19 @@ def prerelease_deps(session): system_test_folder_path = os.path.join("tests", "system") # Only run system tests if found. - if os.path.exists(system_test_path) or os.path.exists(system_test_folder_path): - session.run("py.test", "tests/system") - - snippets_test_path = os.path.join("samples", "snippets") - - # Only run samples tests if found. - if os.path.exists(snippets_test_path): - session.run("py.test", "samples/snippets") + if os.path.exists(system_test_path): + session.run( + "py.test", + "--verbose", + f"--junitxml=system_{session.python}_sponge_log.xml", + system_test_path, + *session.posargs, + ) + if os.path.exists(system_test_folder_path): + session.run( + "py.test", + "--verbose", + f"--junitxml=system_{session.python}_sponge_log.xml", + system_test_folder_path, + *session.posargs, + ) From 41249ebe43e6763beb7c518ab9792b41367cb383 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Sat, 6 Aug 2022 04:04:47 +0200 Subject: [PATCH 312/519] chore(deps): update all dependencies (#548) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update all dependencies * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot --- packages/pandas-gbq/samples/snippets/requirements-test.txt | 2 +- packages/pandas-gbq/samples/snippets/requirements.txt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements-test.txt b/packages/pandas-gbq/samples/snippets/requirements-test.txt index 46ad2b178732..05fefa400b3c 100644 --- a/packages/pandas-gbq/samples/snippets/requirements-test.txt +++ b/packages/pandas-gbq/samples/snippets/requirements-test.txt @@ -1,2 +1,2 @@ -google-cloud-testutils==1.3.2 +google-cloud-testutils==1.3.3 pytest==7.1.2 diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 57a896327693..2442acce3a48 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,5 +1,5 @@ -google-cloud-bigquery-storage==2.13.2 -google-cloud-bigquery==3.1.0 +google-cloud-bigquery-storage==2.14.1 +google-cloud-bigquery==3.3.0 pandas===1.3.5; python_version == '3.7' -pandas==1.4.2; python_version >= '3.8' +pandas==1.4.3; python_version >= '3.8' pyarrow==8.0.0 From 12641b8dfe79ae498a51b473031b0a55ac497164 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Tue, 9 Aug 2022 13:00:04 +0200 Subject: [PATCH 313/519] fix(deps): allow pyarrow < 10 (#550) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update all dependencies * fix(deps): allow pyarrow < 10 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * chore: update dependency google-cloud-bigquery==3.3.1 Co-authored-by: Anthonios Partheniou Co-authored-by: Owl Bot --- packages/pandas-gbq/samples/snippets/requirements.txt | 4 ++-- packages/pandas-gbq/setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 2442acce3a48..60f65561e63b 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,5 +1,5 @@ google-cloud-bigquery-storage==2.14.1 -google-cloud-bigquery==3.3.0 +google-cloud-bigquery==3.3.1 pandas===1.3.5; python_version == '3.7' pandas==1.4.3; python_version >= '3.8' -pyarrow==8.0.0 +pyarrow==9.0.0 diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index c070782d59d6..0bf0c7b26c7c 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -26,7 +26,7 @@ "db-dtypes >=0.3.1,<2.0.0", "numpy >=1.16.6", "pandas >=0.24.2", - "pyarrow >=3.0.0, <9.0dev", + "pyarrow >=3.0.0, <10.0dev", "pydata-google-auth", # Note: google-api-core and google-auth are also included via transitive # dependency on google-cloud-bigquery, but this library also uses them From 485822c18ef0534dd3c2c355028cdbfa9cb0f8aa Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 9 Aug 2022 07:42:10 -0400 Subject: [PATCH 314/519] chore(main): release 0.17.8 (#551) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- packages/pandas-gbq/CHANGELOG.md | 7 +++++++ packages/pandas-gbq/pandas_gbq/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index d5073a0ab702..da823a74f3ff 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.17.8](https://github.com/googleapis/python-bigquery-pandas/compare/v0.17.7...v0.17.8) (2022-08-09) + + +### Bug Fixes + +* **deps:** allow pyarrow < 10 ([#550](https://github.com/googleapis/python-bigquery-pandas/issues/550)) ([c21a414](https://github.com/googleapis/python-bigquery-pandas/commit/c21a41425d4d06b9df5fbdeb15e90f79de841612)) + ## [0.17.7](https://github.com/googleapis/python-bigquery-pandas/compare/v0.17.6...v0.17.7) (2022-07-11) diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py index 63a5c240fac0..dec39535a47b 100644 --- a/packages/pandas-gbq/pandas_gbq/version.py +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -2,4 +2,4 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -__version__ = "0.17.7" +__version__ = "0.17.8" From 5e144b25bf400b487cd9e05a1f25a8fc51d7fd23 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 9 Aug 2022 20:46:22 -0400 Subject: [PATCH 315/519] chore(deps): update actions/setup-python action to v4 [autoapprove] (#553) Source-Link: https://github.com/googleapis/synthtool/commit/8e55b327bae44b6640c7ab4be91df85fc4d6fe8a Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:c6c965a4bf40c19011b11f87dbc801a66d3a23fbc6704102be064ef31c51f1c3 Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 ++-- packages/pandas-gbq/.github/workflows/docs.yml | 4 ++-- packages/pandas-gbq/.github/workflows/lint.yml | 2 +- packages/pandas-gbq/.github/workflows/unittest.yml | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 0eb02fda4c09..c701359fc58c 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:9db98b055a7f8bd82351238ccaacfd3cda58cdf73012ab58b8da146368330021 -# created: 2022-07-25T16:02:49.174178716Z + digest: sha256:c6c965a4bf40c19011b11f87dbc801a66d3a23fbc6704102be064ef31c51f1c3 +# created: 2022-08-09T15:58:56.463048506Z diff --git a/packages/pandas-gbq/.github/workflows/docs.yml b/packages/pandas-gbq/.github/workflows/docs.yml index b46d7305d8cf..7092a139aed3 100644 --- a/packages/pandas-gbq/.github/workflows/docs.yml +++ b/packages/pandas-gbq/.github/workflows/docs.yml @@ -10,7 +10,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: "3.10" - name: Install nox @@ -26,7 +26,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: "3.10" - name: Install nox diff --git a/packages/pandas-gbq/.github/workflows/lint.yml b/packages/pandas-gbq/.github/workflows/lint.yml index f512a4960beb..d2aee5b7d8ec 100644 --- a/packages/pandas-gbq/.github/workflows/lint.yml +++ b/packages/pandas-gbq/.github/workflows/lint.yml @@ -10,7 +10,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: "3.10" - name: Install nox diff --git a/packages/pandas-gbq/.github/workflows/unittest.yml b/packages/pandas-gbq/.github/workflows/unittest.yml index c01a911337e9..19d1f4d663b7 100644 --- a/packages/pandas-gbq/.github/workflows/unittest.yml +++ b/packages/pandas-gbq/.github/workflows/unittest.yml @@ -13,7 +13,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} - name: Install nox @@ -39,7 +39,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: "3.10" - name: Install coverage From 6372a4135baf16acf264647c87ddd8762e640af9 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 1 Sep 2022 14:31:47 -0400 Subject: [PATCH 316/519] chore: remove 'pip install' statements from python_library templates (#563) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(python): exclude `grpcio==1.49.0rc1` in tests Source-Link: https://github.com/googleapis/synthtool/commit/c4dd5953003d13b239f872d329c3146586bb417e Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:ce3c1686bc81145c81dd269bd12c4025c6b275b22d14641358827334fddb1d72 * exclude grpc 1.49.0rc1 in tests * use latest post processor image * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot Co-authored-by: Anthonios Partheniou --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 +- packages/pandas-gbq/.kokoro/publish-docs.sh | 4 +- packages/pandas-gbq/.kokoro/release.sh | 5 +- packages/pandas-gbq/.kokoro/requirements.in | 8 + packages/pandas-gbq/.kokoro/requirements.txt | 464 ++++++++++++++++++ packages/pandas-gbq/noxfile.py | 10 +- packages/pandas-gbq/owlbot.py | 3 +- packages/pandas-gbq/renovate.json | 2 +- 8 files changed, 486 insertions(+), 14 deletions(-) create mode 100644 packages/pandas-gbq/.kokoro/requirements.in create mode 100644 packages/pandas-gbq/.kokoro/requirements.txt diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index c701359fc58c..cf735847a86a 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:c6c965a4bf40c19011b11f87dbc801a66d3a23fbc6704102be064ef31c51f1c3 -# created: 2022-08-09T15:58:56.463048506Z + digest: sha256:1f0dbd02745fb7cf255563dab5968345989308544e52b7f460deadd5e78e63b0 +# created: 2022-08-29T17:28:30.441852797Z diff --git a/packages/pandas-gbq/.kokoro/publish-docs.sh b/packages/pandas-gbq/.kokoro/publish-docs.sh index 8acb14e802b0..1c4d62370042 100755 --- a/packages/pandas-gbq/.kokoro/publish-docs.sh +++ b/packages/pandas-gbq/.kokoro/publish-docs.sh @@ -21,14 +21,12 @@ export PYTHONUNBUFFERED=1 export PATH="${HOME}/.local/bin:${PATH}" # Install nox -python3 -m pip install --user --upgrade --quiet nox +python3 -m pip install --require-hashes -r .kokoro/requirements.txt python3 -m nox --version # build docs nox -s docs -python3 -m pip install --user gcp-docuploader - # create metadata python3 -m docuploader create-metadata \ --name=$(jq --raw-output '.name // empty' .repo-metadata.json) \ diff --git a/packages/pandas-gbq/.kokoro/release.sh b/packages/pandas-gbq/.kokoro/release.sh index 36e4e043d924..56d7f68565e1 100755 --- a/packages/pandas-gbq/.kokoro/release.sh +++ b/packages/pandas-gbq/.kokoro/release.sh @@ -16,12 +16,9 @@ set -eo pipefail # Start the releasetool reporter -python3 -m pip install gcp-releasetool +python3 -m pip install --require-hashes -r github/python-bigquery-pandas/.kokoro/requirements.txt python3 -m releasetool publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script -# Ensure that we have the latest versions of Twine, Wheel, and Setuptools. -python3 -m pip install --upgrade twine wheel setuptools - # Disable buffering, so that the logs stream through. export PYTHONUNBUFFERED=1 diff --git a/packages/pandas-gbq/.kokoro/requirements.in b/packages/pandas-gbq/.kokoro/requirements.in new file mode 100644 index 000000000000..7718391a34d7 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/requirements.in @@ -0,0 +1,8 @@ +gcp-docuploader +gcp-releasetool +importlib-metadata +typing-extensions +twine +wheel +setuptools +nox \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/requirements.txt b/packages/pandas-gbq/.kokoro/requirements.txt new file mode 100644 index 000000000000..92b2f727e777 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/requirements.txt @@ -0,0 +1,464 @@ +# +# This file is autogenerated by pip-compile with python 3.10 +# To update, run: +# +# pip-compile --allow-unsafe --generate-hashes requirements.in +# +argcomplete==2.0.0 \ + --hash=sha256:6372ad78c89d662035101418ae253668445b391755cfe94ea52f1b9d22425b20 \ + --hash=sha256:cffa11ea77999bb0dd27bb25ff6dc142a6796142f68d45b1a26b11f58724561e + # via nox +attrs==22.1.0 \ + --hash=sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6 \ + --hash=sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c + # via gcp-releasetool +bleach==5.0.1 \ + --hash=sha256:085f7f33c15bd408dd9b17a4ad77c577db66d76203e5984b1bd59baeee948b2a \ + --hash=sha256:0d03255c47eb9bd2f26aa9bb7f2107732e7e8fe195ca2f64709fcf3b0a4a085c + # via readme-renderer +cachetools==5.2.0 \ + --hash=sha256:6a94c6402995a99c3970cc7e4884bb60b4a8639938157eeed436098bf9831757 \ + --hash=sha256:f9f17d2aec496a9aa6b76f53e3b614c965223c061982d434d160f930c698a9db + # via google-auth +certifi==2022.6.15 \ + --hash=sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d \ + --hash=sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412 + # via requests +cffi==1.15.1 \ + --hash=sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5 \ + --hash=sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef \ + --hash=sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104 \ + --hash=sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426 \ + --hash=sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405 \ + --hash=sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375 \ + --hash=sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a \ + --hash=sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e \ + --hash=sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc \ + --hash=sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf \ + --hash=sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185 \ + --hash=sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497 \ + --hash=sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3 \ + --hash=sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35 \ + --hash=sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c \ + --hash=sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83 \ + --hash=sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21 \ + --hash=sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca \ + --hash=sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984 \ + --hash=sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac \ + --hash=sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd \ + --hash=sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee \ + --hash=sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a \ + --hash=sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2 \ + --hash=sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192 \ + --hash=sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7 \ + --hash=sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585 \ + --hash=sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f \ + --hash=sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e \ + --hash=sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27 \ + --hash=sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b \ + --hash=sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e \ + --hash=sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e \ + --hash=sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d \ + --hash=sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c \ + --hash=sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415 \ + --hash=sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82 \ + --hash=sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02 \ + --hash=sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314 \ + --hash=sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325 \ + --hash=sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c \ + --hash=sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3 \ + --hash=sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914 \ + --hash=sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045 \ + --hash=sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d \ + --hash=sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9 \ + --hash=sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5 \ + --hash=sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2 \ + --hash=sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c \ + --hash=sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3 \ + --hash=sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2 \ + --hash=sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8 \ + --hash=sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d \ + --hash=sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d \ + --hash=sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9 \ + --hash=sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162 \ + --hash=sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76 \ + --hash=sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4 \ + --hash=sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e \ + --hash=sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9 \ + --hash=sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6 \ + --hash=sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b \ + --hash=sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01 \ + --hash=sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0 + # via cryptography +charset-normalizer==2.1.1 \ + --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 \ + --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f + # via requests +click==8.0.4 \ + --hash=sha256:6a7a62563bbfabfda3a38f3023a1db4a35978c0abd76f6c9605ecd6554d6d9b1 \ + --hash=sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb + # via + # gcp-docuploader + # gcp-releasetool +colorlog==6.7.0 \ + --hash=sha256:0d33ca236784a1ba3ff9c532d4964126d8a2c44f1f0cb1d2b0728196f512f662 \ + --hash=sha256:bd94bd21c1e13fac7bd3153f4bc3a7dc0eb0974b8bc2fdf1a989e474f6e582e5 + # via + # gcp-docuploader + # nox +commonmark==0.9.1 \ + --hash=sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60 \ + --hash=sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9 + # via rich +cryptography==37.0.4 \ + --hash=sha256:190f82f3e87033821828f60787cfa42bff98404483577b591429ed99bed39d59 \ + --hash=sha256:2be53f9f5505673eeda5f2736bea736c40f051a739bfae2f92d18aed1eb54596 \ + --hash=sha256:30788e070800fec9bbcf9faa71ea6d8068f5136f60029759fd8c3efec3c9dcb3 \ + --hash=sha256:3d41b965b3380f10e4611dbae366f6dc3cefc7c9ac4e8842a806b9672ae9add5 \ + --hash=sha256:4c590ec31550a724ef893c50f9a97a0c14e9c851c85621c5650d699a7b88f7ab \ + --hash=sha256:549153378611c0cca1042f20fd9c5030d37a72f634c9326e225c9f666d472884 \ + --hash=sha256:63f9c17c0e2474ccbebc9302ce2f07b55b3b3fcb211ded18a42d5764f5c10a82 \ + --hash=sha256:6bc95ed67b6741b2607298f9ea4932ff157e570ef456ef7ff0ef4884a134cc4b \ + --hash=sha256:7099a8d55cd49b737ffc99c17de504f2257e3787e02abe6d1a6d136574873441 \ + --hash=sha256:75976c217f10d48a8b5a8de3d70c454c249e4b91851f6838a4e48b8f41eb71aa \ + --hash=sha256:7bc997818309f56c0038a33b8da5c0bfbb3f1f067f315f9abd6fc07ad359398d \ + --hash=sha256:80f49023dd13ba35f7c34072fa17f604d2f19bf0989f292cedf7ab5770b87a0b \ + --hash=sha256:91ce48d35f4e3d3f1d83e29ef4a9267246e6a3be51864a5b7d2247d5086fa99a \ + --hash=sha256:a958c52505c8adf0d3822703078580d2c0456dd1d27fabfb6f76fe63d2971cd6 \ + --hash=sha256:b62439d7cd1222f3da897e9a9fe53bbf5c104fff4d60893ad1355d4c14a24157 \ + --hash=sha256:b7f8dd0d4c1f21759695c05a5ec8536c12f31611541f8904083f3dc582604280 \ + --hash=sha256:d204833f3c8a33bbe11eda63a54b1aad7aa7456ed769a982f21ec599ba5fa282 \ + --hash=sha256:e007f052ed10cc316df59bc90fbb7ff7950d7e2919c9757fd42a2b8ecf8a5f67 \ + --hash=sha256:f2dcb0b3b63afb6df7fd94ec6fbddac81b5492513f7b0436210d390c14d46ee8 \ + --hash=sha256:f721d1885ecae9078c3f6bbe8a88bc0786b6e749bf32ccec1ef2b18929a05046 \ + --hash=sha256:f7a6de3e98771e183645181b3627e2563dcde3ce94a9e42a3f427d2255190327 \ + --hash=sha256:f8c0a6e9e1dd3eb0414ba320f85da6b0dcbd543126e30fcc546e7372a7fbf3b9 + # via + # gcp-releasetool + # secretstorage +distlib==0.3.6 \ + --hash=sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46 \ + --hash=sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e + # via virtualenv +docutils==0.19 \ + --hash=sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6 \ + --hash=sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc + # via readme-renderer +filelock==3.8.0 \ + --hash=sha256:55447caa666f2198c5b6b13a26d2084d26fa5b115c00d065664b2124680c4edc \ + --hash=sha256:617eb4e5eedc82fc5f47b6d61e4d11cb837c56cb4544e39081099fa17ad109d4 + # via virtualenv +gcp-docuploader==0.6.3 \ + --hash=sha256:ba8c9d76b3bbac54b0311c503a373b00edc2dc02d6d54ea9507045adb8e870f7 \ + --hash=sha256:c0f5aaa82ce1854a386197e4e359b120ad6d4e57ae2c812fce42219a3288026b + # via -r requirements.in +gcp-releasetool==1.8.7 \ + --hash=sha256:3d2a67c9db39322194afb3b427e9cb0476ce8f2a04033695f0aeb63979fc2b37 \ + --hash=sha256:5e4d28f66e90780d77f3ecf1e9155852b0c3b13cbccb08ab07e66b2357c8da8d + # via -r requirements.in +google-api-core==2.8.2 \ + --hash=sha256:06f7244c640322b508b125903bb5701bebabce8832f85aba9335ec00b3d02edc \ + --hash=sha256:93c6a91ccac79079ac6bbf8b74ee75db970cc899278b97d53bc012f35908cf50 + # via + # google-cloud-core + # google-cloud-storage +google-auth==2.11.0 \ + --hash=sha256:be62acaae38d0049c21ca90f27a23847245c9f161ff54ede13af2cb6afecbac9 \ + --hash=sha256:ed65ecf9f681832298e29328e1ef0a3676e3732b2e56f41532d45f70a22de0fb + # via + # gcp-releasetool + # google-api-core + # google-cloud-core + # google-cloud-storage +google-cloud-core==2.3.2 \ + --hash=sha256:8417acf6466be2fa85123441696c4badda48db314c607cf1e5d543fa8bdc22fe \ + --hash=sha256:b9529ee7047fd8d4bf4a2182de619154240df17fbe60ead399078c1ae152af9a + # via google-cloud-storage +google-cloud-storage==2.5.0 \ + --hash=sha256:19a26c66c317ce542cea0830b7e787e8dac2588b6bfa4d3fd3b871ba16305ab0 \ + --hash=sha256:382f34b91de2212e3c2e7b40ec079d27ee2e3dbbae99b75b1bcd8c63063ce235 + # via gcp-docuploader +google-crc32c==1.3.0 \ + --hash=sha256:04e7c220798a72fd0f08242bc8d7a05986b2a08a0573396187fd32c1dcdd58b3 \ + --hash=sha256:05340b60bf05b574159e9bd940152a47d38af3fb43803ffe71f11d704b7696a6 \ + --hash=sha256:12674a4c3b56b706153a358eaa1018c4137a5a04635b92b4652440d3d7386206 \ + --hash=sha256:127f9cc3ac41b6a859bd9dc4321097b1a4f6aa7fdf71b4f9227b9e3ebffb4422 \ + --hash=sha256:13af315c3a0eec8bb8b8d80b8b128cb3fcd17d7e4edafc39647846345a3f003a \ + --hash=sha256:1926fd8de0acb9d15ee757175ce7242e235482a783cd4ec711cc999fc103c24e \ + --hash=sha256:226f2f9b8e128a6ca6a9af9b9e8384f7b53a801907425c9a292553a3a7218ce0 \ + --hash=sha256:276de6273eb074a35bc598f8efbc00c7869c5cf2e29c90748fccc8c898c244df \ + --hash=sha256:318f73f5484b5671f0c7f5f63741ab020a599504ed81d209b5c7129ee4667407 \ + --hash=sha256:3bbce1be3687bbfebe29abdb7631b83e6b25da3f4e1856a1611eb21854b689ea \ + --hash=sha256:42ae4781333e331a1743445931b08ebdad73e188fd554259e772556fc4937c48 \ + --hash=sha256:58be56ae0529c664cc04a9c76e68bb92b091e0194d6e3c50bea7e0f266f73713 \ + --hash=sha256:5da2c81575cc3ccf05d9830f9e8d3c70954819ca9a63828210498c0774fda1a3 \ + --hash=sha256:6311853aa2bba4064d0c28ca54e7b50c4d48e3de04f6770f6c60ebda1e975267 \ + --hash=sha256:650e2917660e696041ab3dcd7abac160b4121cd9a484c08406f24c5964099829 \ + --hash=sha256:6a4db36f9721fdf391646685ecffa404eb986cbe007a3289499020daf72e88a2 \ + --hash=sha256:779cbf1ce375b96111db98fca913c1f5ec11b1d870e529b1dc7354b2681a8c3a \ + --hash=sha256:7f6fe42536d9dcd3e2ffb9d3053f5d05221ae3bbcefbe472bdf2c71c793e3183 \ + --hash=sha256:891f712ce54e0d631370e1f4997b3f182f3368179198efc30d477c75d1f44942 \ + --hash=sha256:95c68a4b9b7828ba0428f8f7e3109c5d476ca44996ed9a5f8aac6269296e2d59 \ + --hash=sha256:96a8918a78d5d64e07c8ea4ed2bc44354e3f93f46a4866a40e8db934e4c0d74b \ + --hash=sha256:9c3cf890c3c0ecfe1510a452a165431b5831e24160c5fcf2071f0f85ca5a47cd \ + --hash=sha256:9f58099ad7affc0754ae42e6d87443299f15d739b0ce03c76f515153a5cda06c \ + --hash=sha256:a0b9e622c3b2b8d0ce32f77eba617ab0d6768b82836391e4f8f9e2074582bf02 \ + --hash=sha256:a7f9cbea4245ee36190f85fe1814e2d7b1e5f2186381b082f5d59f99b7f11328 \ + --hash=sha256:bab4aebd525218bab4ee615786c4581952eadc16b1ff031813a2fd51f0cc7b08 \ + --hash=sha256:c124b8c8779bf2d35d9b721e52d4adb41c9bfbde45e6a3f25f0820caa9aba73f \ + --hash=sha256:c9da0a39b53d2fab3e5467329ed50e951eb91386e9d0d5b12daf593973c3b168 \ + --hash=sha256:ca60076c388728d3b6ac3846842474f4250c91efbfe5afa872d3ffd69dd4b318 \ + --hash=sha256:cb6994fff247987c66a8a4e550ef374671c2b82e3c0d2115e689d21e511a652d \ + --hash=sha256:d1c1d6236feab51200272d79b3d3e0f12cf2cbb12b208c835b175a21efdb0a73 \ + --hash=sha256:dd7760a88a8d3d705ff562aa93f8445ead54f58fd482e4f9e2bafb7e177375d4 \ + --hash=sha256:dda4d8a3bb0b50f540f6ff4b6033f3a74e8bf0bd5320b70fab2c03e512a62812 \ + --hash=sha256:e0f1ff55dde0ebcfbef027edc21f71c205845585fffe30d4ec4979416613e9b3 \ + --hash=sha256:e7a539b9be7b9c00f11ef16b55486141bc2cdb0c54762f84e3c6fc091917436d \ + --hash=sha256:eb0b14523758e37802f27b7f8cd973f5f3d33be7613952c0df904b68c4842f0e \ + --hash=sha256:ed447680ff21c14aaceb6a9f99a5f639f583ccfe4ce1a5e1d48eb41c3d6b3217 \ + --hash=sha256:f52a4ad2568314ee713715b1e2d79ab55fab11e8b304fd1462ff5cccf4264b3e \ + --hash=sha256:fbd60c6aaa07c31d7754edbc2334aef50601b7f1ada67a96eb1eb57c7c72378f \ + --hash=sha256:fc28e0db232c62ca0c3600884933178f0825c99be4474cdd645e378a10588125 \ + --hash=sha256:fe31de3002e7b08eb20823b3735b97c86c5926dd0581c7710a680b418a8709d4 \ + --hash=sha256:fec221a051150eeddfdfcff162e6db92c65ecf46cb0f7bb1bf812a1520ec026b \ + --hash=sha256:ff71073ebf0e42258a42a0b34f2c09ec384977e7f6808999102eedd5b49920e3 + # via google-resumable-media +google-resumable-media==2.3.3 \ + --hash=sha256:27c52620bd364d1c8116eaac4ea2afcbfb81ae9139fb3199652fcac1724bfb6c \ + --hash=sha256:5b52774ea7a829a8cdaa8bd2d4c3d4bc660c91b30857ab2668d0eb830f4ea8c5 + # via google-cloud-storage +googleapis-common-protos==1.56.4 \ + --hash=sha256:8eb2cbc91b69feaf23e32452a7ae60e791e09967d81d4fcc7fc388182d1bd394 \ + --hash=sha256:c25873c47279387cfdcbdafa36149887901d36202cb645a0e4f29686bf6e4417 + # via google-api-core +idna==3.3 \ + --hash=sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff \ + --hash=sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d + # via requests +importlib-metadata==4.12.0 \ + --hash=sha256:637245b8bab2b6502fcbc752cc4b7a6f6243bb02b31c5c26156ad103d3d45670 \ + --hash=sha256:7401a975809ea1fdc658c3aa4f78cc2195a0e019c5cbc4c06122884e9ae80c23 + # via + # -r requirements.in + # twine +jeepney==0.8.0 \ + --hash=sha256:5efe48d255973902f6badc3ce55e2aa6c5c3b3bc642059ef3a91247bcfcc5806 \ + --hash=sha256:c0a454ad016ca575060802ee4d590dd912e35c122fa04e70306de3d076cce755 + # via + # keyring + # secretstorage +jinja2==3.1.2 \ + --hash=sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852 \ + --hash=sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61 + # via gcp-releasetool +keyring==23.9.0 \ + --hash=sha256:4c32a31174faaee48f43a7e2c7e9c3216ec5e95acf22a2bebfb4a1d05056ee44 \ + --hash=sha256:98f060ec95ada2ab910c195a2d4317be6ef87936a766b239c46aa3c7aac4f0db + # via + # gcp-releasetool + # twine +markupsafe==2.1.1 \ + --hash=sha256:0212a68688482dc52b2d45013df70d169f542b7394fc744c02a57374a4207003 \ + --hash=sha256:089cf3dbf0cd6c100f02945abeb18484bd1ee57a079aefd52cffd17fba910b88 \ + --hash=sha256:10c1bfff05d95783da83491be968e8fe789263689c02724e0c691933c52994f5 \ + --hash=sha256:33b74d289bd2f5e527beadcaa3f401e0df0a89927c1559c8566c066fa4248ab7 \ + --hash=sha256:3799351e2336dc91ea70b034983ee71cf2f9533cdff7c14c90ea126bfd95d65a \ + --hash=sha256:3ce11ee3f23f79dbd06fb3d63e2f6af7b12db1d46932fe7bd8afa259a5996603 \ + --hash=sha256:421be9fbf0ffe9ffd7a378aafebbf6f4602d564d34be190fc19a193232fd12b1 \ + --hash=sha256:43093fb83d8343aac0b1baa75516da6092f58f41200907ef92448ecab8825135 \ + --hash=sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247 \ + --hash=sha256:4a33dea2b688b3190ee12bd7cfa29d39c9ed176bda40bfa11099a3ce5d3a7ac6 \ + --hash=sha256:4b9fe39a2ccc108a4accc2676e77da025ce383c108593d65cc909add5c3bd601 \ + --hash=sha256:56442863ed2b06d19c37f94d999035e15ee982988920e12a5b4ba29b62ad1f77 \ + --hash=sha256:671cd1187ed5e62818414afe79ed29da836dde67166a9fac6d435873c44fdd02 \ + --hash=sha256:694deca8d702d5db21ec83983ce0bb4b26a578e71fbdbd4fdcd387daa90e4d5e \ + --hash=sha256:6a074d34ee7a5ce3effbc526b7083ec9731bb3cbf921bbe1d3005d4d2bdb3a63 \ + --hash=sha256:6d0072fea50feec76a4c418096652f2c3238eaa014b2f94aeb1d56a66b41403f \ + --hash=sha256:6fbf47b5d3728c6aea2abb0589b5d30459e369baa772e0f37a0320185e87c980 \ + --hash=sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b \ + --hash=sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812 \ + --hash=sha256:8dc1c72a69aa7e082593c4a203dcf94ddb74bb5c8a731e4e1eb68d031e8498ff \ + --hash=sha256:8e3dcf21f367459434c18e71b2a9532d96547aef8a871872a5bd69a715c15f96 \ + --hash=sha256:8e576a51ad59e4bfaac456023a78f6b5e6e7651dcd383bcc3e18d06f9b55d6d1 \ + --hash=sha256:96e37a3dc86e80bf81758c152fe66dbf60ed5eca3d26305edf01892257049925 \ + --hash=sha256:97a68e6ada378df82bc9f16b800ab77cbf4b2fada0081794318520138c088e4a \ + --hash=sha256:99a2a507ed3ac881b975a2976d59f38c19386d128e7a9a18b7df6fff1fd4c1d6 \ + --hash=sha256:a49907dd8420c5685cfa064a1335b6754b74541bbb3706c259c02ed65b644b3e \ + --hash=sha256:b09bf97215625a311f669476f44b8b318b075847b49316d3e28c08e41a7a573f \ + --hash=sha256:b7bd98b796e2b6553da7225aeb61f447f80a1ca64f41d83612e6139ca5213aa4 \ + --hash=sha256:b87db4360013327109564f0e591bd2a3b318547bcef31b468a92ee504d07ae4f \ + --hash=sha256:bcb3ed405ed3222f9904899563d6fc492ff75cce56cba05e32eff40e6acbeaa3 \ + --hash=sha256:d4306c36ca495956b6d568d276ac11fdd9c30a36f1b6eb928070dc5360b22e1c \ + --hash=sha256:d5ee4f386140395a2c818d149221149c54849dfcfcb9f1debfe07a8b8bd63f9a \ + --hash=sha256:dda30ba7e87fbbb7eab1ec9f58678558fd9a6b8b853530e176eabd064da81417 \ + --hash=sha256:e04e26803c9c3851c931eac40c695602c6295b8d432cbe78609649ad9bd2da8a \ + --hash=sha256:e1c0b87e09fa55a220f058d1d49d3fb8df88fbfab58558f1198e08c1e1de842a \ + --hash=sha256:e72591e9ecd94d7feb70c1cbd7be7b3ebea3f548870aa91e2732960fa4d57a37 \ + --hash=sha256:e8c843bbcda3a2f1e3c2ab25913c80a3c5376cd00c6e8c4a86a89a28c8dc5452 \ + --hash=sha256:efc1913fd2ca4f334418481c7e595c00aad186563bbc1ec76067848c7ca0a933 \ + --hash=sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a \ + --hash=sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7 + # via jinja2 +nox==2022.8.7 \ + --hash=sha256:1b894940551dc5c389f9271d197ca5d655d40bdc6ccf93ed6880e4042760a34b \ + --hash=sha256:96cca88779e08282a699d672258ec01eb7c792d35bbbf538c723172bce23212c + # via -r requirements.in +packaging==21.3 \ + --hash=sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb \ + --hash=sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522 + # via + # gcp-releasetool + # nox +pkginfo==1.8.3 \ + --hash=sha256:848865108ec99d4901b2f7e84058b6e7660aae8ae10164e015a6dcf5b242a594 \ + --hash=sha256:a84da4318dd86f870a9447a8c98340aa06216bfc6f2b7bdc4b8766984ae1867c + # via twine +platformdirs==2.5.2 \ + --hash=sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788 \ + --hash=sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19 + # via virtualenv +protobuf==3.20.1 \ + --hash=sha256:06059eb6953ff01e56a25cd02cca1a9649a75a7e65397b5b9b4e929ed71d10cf \ + --hash=sha256:097c5d8a9808302fb0da7e20edf0b8d4703274d140fd25c5edabddcde43e081f \ + --hash=sha256:284f86a6207c897542d7e956eb243a36bb8f9564c1742b253462386e96c6b78f \ + --hash=sha256:32ca378605b41fd180dfe4e14d3226386d8d1b002ab31c969c366549e66a2bb7 \ + --hash=sha256:3cc797c9d15d7689ed507b165cd05913acb992d78b379f6014e013f9ecb20996 \ + --hash=sha256:62f1b5c4cd6c5402b4e2d63804ba49a327e0c386c99b1675c8a0fefda23b2067 \ + --hash=sha256:69ccfdf3657ba59569c64295b7d51325f91af586f8d5793b734260dfe2e94e2c \ + --hash=sha256:6f50601512a3d23625d8a85b1638d914a0970f17920ff39cec63aaef80a93fb7 \ + --hash=sha256:7403941f6d0992d40161aa8bb23e12575637008a5a02283a930addc0508982f9 \ + --hash=sha256:755f3aee41354ae395e104d62119cb223339a8f3276a0cd009ffabfcdd46bb0c \ + --hash=sha256:77053d28427a29987ca9caf7b72ccafee011257561259faba8dd308fda9a8739 \ + --hash=sha256:7e371f10abe57cee5021797126c93479f59fccc9693dafd6bd5633ab67808a91 \ + --hash=sha256:9016d01c91e8e625141d24ec1b20fed584703e527d28512aa8c8707f105a683c \ + --hash=sha256:9be73ad47579abc26c12024239d3540e6b765182a91dbc88e23658ab71767153 \ + --hash=sha256:adc31566d027f45efe3f44eeb5b1f329da43891634d61c75a5944e9be6dd42c9 \ + --hash=sha256:adfc6cf69c7f8c50fd24c793964eef18f0ac321315439d94945820612849c388 \ + --hash=sha256:af0ebadc74e281a517141daad9d0f2c5d93ab78e9d455113719a45a49da9db4e \ + --hash=sha256:cb29edb9eab15742d791e1025dd7b6a8f6fcb53802ad2f6e3adcb102051063ab \ + --hash=sha256:cd68be2559e2a3b84f517fb029ee611546f7812b1fdd0aa2ecc9bc6ec0e4fdde \ + --hash=sha256:cdee09140e1cd184ba9324ec1df410e7147242b94b5f8b0c64fc89e38a8ba531 \ + --hash=sha256:db977c4ca738dd9ce508557d4fce0f5aebd105e158c725beec86feb1f6bc20d8 \ + --hash=sha256:dd5789b2948ca702c17027c84c2accb552fc30f4622a98ab5c51fcfe8c50d3e7 \ + --hash=sha256:e250a42f15bf9d5b09fe1b293bdba2801cd520a9f5ea2d7fb7536d4441811d20 \ + --hash=sha256:ff8d8fa42675249bb456f5db06c00de6c2f4c27a065955917b28c4f15978b9c3 + # via + # gcp-docuploader + # gcp-releasetool + # google-api-core +py==1.11.0 \ + --hash=sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719 \ + --hash=sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378 + # via nox +pyasn1==0.4.8 \ + --hash=sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d \ + --hash=sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.2.8 \ + --hash=sha256:905f84c712230b2c592c19470d3ca8d552de726050d1d1716282a1f6146be65e \ + --hash=sha256:a50b808ffeb97cb3601dd25981f6b016cbb3d31fbf57a8b8a87428e6158d0c74 + # via google-auth +pycparser==2.21 \ + --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ + --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206 + # via cffi +pygments==2.13.0 \ + --hash=sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1 \ + --hash=sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42 + # via + # readme-renderer + # rich +pyjwt==2.4.0 \ + --hash=sha256:72d1d253f32dbd4f5c88eaf1fdc62f3a19f676ccbadb9dbc5d07e951b2b26daf \ + --hash=sha256:d42908208c699b3b973cbeb01a969ba6a96c821eefb1c5bfe4c390c01d67abba + # via gcp-releasetool +pyparsing==3.0.9 \ + --hash=sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb \ + --hash=sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc + # via packaging +pyperclip==1.8.2 \ + --hash=sha256:105254a8b04934f0bc84e9c24eb360a591aaf6535c9def5f29d92af107a9bf57 + # via gcp-releasetool +python-dateutil==2.8.2 \ + --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ + --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 + # via gcp-releasetool +readme-renderer==37.0 \ + --hash=sha256:07b7ea234e03e58f77cc222e206e6abb8f4c0435becce5104794ee591f9301c5 \ + --hash=sha256:9fa416704703e509eeb900696751c908ddeb2011319d93700d8f18baff887a69 + # via twine +requests==2.28.1 \ + --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983 \ + --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349 + # via + # gcp-releasetool + # google-api-core + # google-cloud-storage + # requests-toolbelt + # twine +requests-toolbelt==0.9.1 \ + --hash=sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f \ + --hash=sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0 + # via twine +rfc3986==2.0.0 \ + --hash=sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd \ + --hash=sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c + # via twine +rich==12.5.1 \ + --hash=sha256:2eb4e6894cde1e017976d2975ac210ef515d7548bc595ba20e195fb9628acdeb \ + --hash=sha256:63a5c5ce3673d3d5fbbf23cd87e11ab84b6b451436f1b7f19ec54b6bc36ed7ca + # via twine +rsa==4.9 \ + --hash=sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7 \ + --hash=sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21 + # via google-auth +secretstorage==3.3.3 \ + --hash=sha256:2403533ef369eca6d2ba81718576c5e0f564d5cca1b58f73a8b23e7d4eeebd77 \ + --hash=sha256:f356e6628222568e3af06f2eba8df495efa13b3b63081dafd4f7d9a7b7bc9f99 + # via keyring +six==1.16.0 \ + --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ + --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 + # via + # bleach + # gcp-docuploader + # google-auth + # python-dateutil +twine==4.0.1 \ + --hash=sha256:42026c18e394eac3e06693ee52010baa5313e4811d5a11050e7d48436cf41b9e \ + --hash=sha256:96b1cf12f7ae611a4a40b6ae8e9570215daff0611828f5fe1f37a16255ab24a0 + # via -r requirements.in +typing-extensions==4.3.0 \ + --hash=sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02 \ + --hash=sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6 + # via -r requirements.in +urllib3==1.26.12 \ + --hash=sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e \ + --hash=sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997 + # via + # requests + # twine +virtualenv==20.16.4 \ + --hash=sha256:014f766e4134d0008dcaa1f95bafa0fb0f575795d07cae50b1bee514185d6782 \ + --hash=sha256:035ed57acce4ac35c82c9d8802202b0e71adac011a511ff650cbcf9635006a22 + # via nox +webencodings==0.5.1 \ + --hash=sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78 \ + --hash=sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923 + # via bleach +wheel==0.37.1 \ + --hash=sha256:4bdcd7d840138086126cd09254dc6195fb4fc6f01c050a1d7236f2630db1d22a \ + --hash=sha256:e9a504e793efbca1b8e0e9cb979a249cf4a0a7b5b8c9e8b65a5e39d49529c1c4 + # via -r requirements.in +zipp==3.8.1 \ + --hash=sha256:05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2 \ + --hash=sha256:47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009 + # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +setuptools==65.2.0 \ + --hash=sha256:7f4bc85450898a09f76ebf28b72fa25bc7111f6c7d665d514a60bba9c75ef2a9 \ + --hash=sha256:a3ca5857c89f82f5c9410e8508cb32f4872a3bafd4aa7ae122a24ca33bccc750 + # via -r requirements.in diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 9ea981f4f0be..07b7345f093f 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -196,7 +196,9 @@ def unit(session): def install_systemtest_dependencies(session, *constraints): # Use pre-release gRPC for system tests. - session.install("--pre", "grpcio") + # Exclude version 1.49.0rc1 which has a known issue. + # See https://github.com/grpc/grpc/pull/30642 + session.install("--pre", "grpcio!=1.49.0rc1") session.install(*SYSTEM_TEST_STANDARD_DEPENDENCIES, *constraints) @@ -292,7 +294,8 @@ def prerelease(session): "google-cloud-bigquery-storage", "google-cloud-core", "google-resumable-media", - "grpcio", + # Exclude version 1.49.0rc1 which has a known issue. See https://github.com/grpc/grpc/pull/30642 + "grpcio!=1.49.0rc1", ) session.install( "freezegun", @@ -459,7 +462,8 @@ def prerelease_deps(session): # dependency of grpc "six", "googleapis-common-protos", - "grpcio", + # Exclude version 1.49.0rc1 which has a known issue. See https://github.com/grpc/grpc/pull/30642 + "grpcio!=1.49.0rc1", "grpcio-status", "google-api-core", "proto-plus", diff --git a/packages/pandas-gbq/owlbot.py b/packages/pandas-gbq/owlbot.py index 1ec8fbe39f81..f5c5c6891440 100644 --- a/packages/pandas-gbq/owlbot.py +++ b/packages/pandas-gbq/owlbot.py @@ -113,7 +113,8 @@ def prerelease(session): "google-cloud-bigquery-storage", "google-cloud-core", "google-resumable-media", - "grpcio", + # Exclude version 1.49.0rc1 which has a known issue. See https://github.com/grpc/grpc/pull/30642 + "grpcio!=1.49.0rc1", ) session.install( "freezegun", diff --git a/packages/pandas-gbq/renovate.json b/packages/pandas-gbq/renovate.json index c21036d385e5..566a70f3cc3c 100644 --- a/packages/pandas-gbq/renovate.json +++ b/packages/pandas-gbq/renovate.json @@ -5,7 +5,7 @@ ":preserveSemverRanges", ":disableDependencyDashboard" ], - "ignorePaths": [".pre-commit-config.yaml"], + "ignorePaths": [".pre-commit-config.yaml", ".kokoro/requirements.txt"], "pip_requirements": { "fileMatch": ["requirements-test.txt", "samples/[\\S/]*constraints.txt", "samples/[\\S/]*constraints-test.txt"] } From 84491a47edf4efea6c33703e2db95fa43170c9a6 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Fri, 2 Sep 2022 06:56:00 -0400 Subject: [PATCH 317/519] test: prevents query cache reuse that seems to trigger dtypes test failure (#557) * fix: updates dataset path * fix: avoid reusing cached results * chore: fixes linting errors * chore: fixes more linting errors * adds comment regarding use of specific parameter to avoid cache mismatch --- packages/pandas-gbq/tests/system/test_read_gbq.py | 8 +++++++- .../tests/system/test_read_gbq_with_bqstorage.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/tests/system/test_read_gbq.py b/packages/pandas-gbq/tests/system/test_read_gbq.py index 0187b69d82e4..f9358ef60bae 100644 --- a/packages/pandas-gbq/tests/system/test_read_gbq.py +++ b/packages/pandas-gbq/tests/system/test_read_gbq.py @@ -547,7 +547,13 @@ def test_default_dtypes( ): if use_bqstorage_api not in use_bqstorage_apis: pytest.skip(f"use_bqstorage_api={use_bqstorage_api} not supported.") - result = read_gbq(query, use_bqstorage_api=use_bqstorage_api) + # the parameter useQueryCache=False is used in the following function call + # to avoid a failing test due to cached data that may be out of order. + result = read_gbq( + query, + use_bqstorage_api=use_bqstorage_api, + configuration={"query": {"useQueryCache": False}}, + ) pandas.testing.assert_frame_equal(result, expected) diff --git a/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py b/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py index 84435b9ff7ce..cfb31ea81a6e 100644 --- a/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py +++ b/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py @@ -50,7 +50,7 @@ def test_large_results(random_dataset, method_under_test): total_amount, passenger_count, trip_distance - FROM `bigquery-public-data.new_york_taxi_trips.tlc_green_trips_2014` + FROM `bigquery-public-data.new_york.tlc_green_trips_2014` -- Select non-null rows for no-copy conversion from Arrow to pandas. WHERE total_amount IS NOT NULL AND passenger_count IS NOT NULL From 5ad52e7e7e0bf7d338f50f8a6ba8cdd09cbc1f16 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 2 Sep 2022 18:40:37 +0000 Subject: [PATCH 318/519] chore(python): exclude setup.py in renovate config (#568) Source-Link: https://github.com/googleapis/synthtool/commit/56da63e80c384a871356d1ea6640802017f213b4 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:993a058718e84a82fda04c3177e58f0a43281a996c7c395e0a56ccc4d6d210d7 --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 3 +-- packages/pandas-gbq/.kokoro/requirements.txt | 8 ++++++++ packages/pandas-gbq/renovate.json | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index cf735847a86a..b8dcb4a4af99 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:1f0dbd02745fb7cf255563dab5968345989308544e52b7f460deadd5e78e63b0 -# created: 2022-08-29T17:28:30.441852797Z + digest: sha256:993a058718e84a82fda04c3177e58f0a43281a996c7c395e0a56ccc4d6d210d7 diff --git a/packages/pandas-gbq/.kokoro/requirements.txt b/packages/pandas-gbq/.kokoro/requirements.txt index 92b2f727e777..385f2d4d6106 100644 --- a/packages/pandas-gbq/.kokoro/requirements.txt +++ b/packages/pandas-gbq/.kokoro/requirements.txt @@ -241,6 +241,10 @@ importlib-metadata==4.12.0 \ # via # -r requirements.in # twine +jaraco-classes==3.2.2 \ + --hash=sha256:6745f113b0b588239ceb49532aa09c3ebb947433ce311ef2f8e3ad64ebb74594 \ + --hash=sha256:e6ef6fd3fcf4579a7a019d87d1e56a883f4e4c35cfe925f86731abc58804e647 + # via keyring jeepney==0.8.0 \ --hash=sha256:5efe48d255973902f6badc3ce55e2aa6c5c3b3bc642059ef3a91247bcfcc5806 \ --hash=sha256:c0a454ad016ca575060802ee4d590dd912e35c122fa04e70306de3d076cce755 @@ -299,6 +303,10 @@ markupsafe==2.1.1 \ --hash=sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a \ --hash=sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7 # via jinja2 +more-itertools==8.14.0 \ + --hash=sha256:1bc4f91ee5b1b31ac7ceacc17c09befe6a40a503907baf9c839c229b5095cfd2 \ + --hash=sha256:c09443cd3d5438b8dafccd867a6bc1cb0894389e90cb53d227456b0b0bccb750 + # via jaraco-classes nox==2022.8.7 \ --hash=sha256:1b894940551dc5c389f9271d197ca5d655d40bdc6ccf93ed6880e4042760a34b \ --hash=sha256:96cca88779e08282a699d672258ec01eb7c792d35bbbf538c723172bce23212c diff --git a/packages/pandas-gbq/renovate.json b/packages/pandas-gbq/renovate.json index 566a70f3cc3c..39b2a0ec9296 100644 --- a/packages/pandas-gbq/renovate.json +++ b/packages/pandas-gbq/renovate.json @@ -5,7 +5,7 @@ ":preserveSemverRanges", ":disableDependencyDashboard" ], - "ignorePaths": [".pre-commit-config.yaml", ".kokoro/requirements.txt"], + "ignorePaths": [".pre-commit-config.yaml", ".kokoro/requirements.txt", "setup.py"], "pip_requirements": { "fileMatch": ["requirements-test.txt", "samples/[\\S/]*constraints.txt", "samples/[\\S/]*constraints-test.txt"] } From b7cd6089f52f91a6ccfcbf32bb498d8ca46703ed Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Tue, 6 Sep 2022 19:53:08 +0200 Subject: [PATCH 319/519] chore(deps): update all dependencies (#567) --- packages/pandas-gbq/samples/snippets/requirements-test.txt | 2 +- packages/pandas-gbq/samples/snippets/requirements.txt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements-test.txt b/packages/pandas-gbq/samples/snippets/requirements-test.txt index 05fefa400b3c..8394ed528160 100644 --- a/packages/pandas-gbq/samples/snippets/requirements-test.txt +++ b/packages/pandas-gbq/samples/snippets/requirements-test.txt @@ -1,2 +1,2 @@ google-cloud-testutils==1.3.3 -pytest==7.1.2 +pytest==7.1.3 diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 60f65561e63b..6c2cb1881908 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,5 +1,5 @@ -google-cloud-bigquery-storage==2.14.1 -google-cloud-bigquery==3.3.1 +google-cloud-bigquery-storage==2.15.0 +google-cloud-bigquery==3.3.2 pandas===1.3.5; python_version == '3.7' -pandas==1.4.3; python_version >= '3.8' +pandas==1.4.4; python_version >= '3.8' pyarrow==9.0.0 From 5fb23f239128702209f8b2971dbe70adb1506678 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 13 Sep 2022 16:32:18 +0000 Subject: [PATCH 320/519] chore: detect samples tests in nested directories (#570) Source-Link: https://github.com/googleapis/synthtool/commit/50db768f450a50d7c1fd62513c113c9bb96fd434 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:e09366bdf0fd9c8976592988390b24d53583dd9f002d476934da43725adbb978 --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 2 +- packages/pandas-gbq/samples/snippets/noxfile.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index b8dcb4a4af99..aa547962eb0a 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:993a058718e84a82fda04c3177e58f0a43281a996c7c395e0a56ccc4d6d210d7 + digest: sha256:e09366bdf0fd9c8976592988390b24d53583dd9f002d476934da43725adbb978 diff --git a/packages/pandas-gbq/samples/snippets/noxfile.py b/packages/pandas-gbq/samples/snippets/noxfile.py index 29b5bc852183..b053ca568f63 100644 --- a/packages/pandas-gbq/samples/snippets/noxfile.py +++ b/packages/pandas-gbq/samples/snippets/noxfile.py @@ -208,8 +208,10 @@ def _session_tests( session: nox.sessions.Session, post_install: Callable = None ) -> None: # check for presence of tests - test_list = glob.glob("*_test.py") + glob.glob("test_*.py") - test_list.extend(glob.glob("tests")) + test_list = glob.glob("**/*_test.py", recursive=True) + glob.glob( + "**/test_*.py", recursive=True + ) + test_list.extend(glob.glob("**/tests", recursive=True)) if len(test_list) == 0: print("No tests found, skipping directory.") From f5138a83f6b6ccdb6b62a760e3ab413db104c7f7 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Tue, 27 Sep 2022 16:59:19 -0400 Subject: [PATCH 321/519] fix: updates requirements.txt to fix failing tests due to missing req (#575) --- packages/pandas-gbq/samples/snippets/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 6c2cb1881908..6a681437d6c9 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,5 +1,6 @@ google-cloud-bigquery-storage==2.15.0 google-cloud-bigquery==3.3.2 +pandas-gbq==0.17.8 pandas===1.3.5; python_version == '3.7' pandas==1.4.4; python_version >= '3.8' pyarrow==9.0.0 From e4b968d5e1bdaa92c172f586485e67df790e2c71 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 28 Sep 2022 15:03:54 -0400 Subject: [PATCH 322/519] chore(main): release 0.17.9 (#576) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- packages/pandas-gbq/CHANGELOG.md | 7 +++++++ packages/pandas-gbq/pandas_gbq/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index da823a74f3ff..c1865ef6fdba 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.17.9](https://github.com/googleapis/python-bigquery-pandas/compare/v0.17.8...v0.17.9) (2022-09-27) + + +### Bug Fixes + +* Updates requirements.txt to fix failing tests due to missing req ([#575](https://github.com/googleapis/python-bigquery-pandas/issues/575)) ([1d797a3](https://github.com/googleapis/python-bigquery-pandas/commit/1d797a3337e716fa6f3de511e7c8875dfadde43b)) + ## [0.17.8](https://github.com/googleapis/python-bigquery-pandas/compare/v0.17.7...v0.17.8) (2022-08-09) diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py index dec39535a47b..8614d84c6b0b 100644 --- a/packages/pandas-gbq/pandas_gbq/version.py +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -2,4 +2,4 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -__version__ = "0.17.8" +__version__ = "0.17.9" From 0b32fe3946971c9152d7a0ceb797a929af514871 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Mon, 3 Oct 2022 12:13:06 -0400 Subject: [PATCH 323/519] chore: update dependency protobuf >= 3.20.2 (#573) * chore: exclude requirements.txt file from renovate-bot Source-Link: https://github.com/googleapis/synthtool/commit/f58d3135a2fab20e225d98741dbc06d57459b816 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:7a40313731a7cb1454eef6b33d3446ebb121836738dc3ab3d2d3ded5268c35b6 * update constraints files Co-authored-by: Owl Bot Co-authored-by: Anthonios Partheniou --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 2 +- packages/pandas-gbq/.kokoro/requirements.txt | 49 +++++++++---------- .../pandas-gbq/testing/constraints-3.7.txt | 2 +- 3 files changed, 26 insertions(+), 27 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index aa547962eb0a..3815c983cb16 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:e09366bdf0fd9c8976592988390b24d53583dd9f002d476934da43725adbb978 + digest: sha256:7a40313731a7cb1454eef6b33d3446ebb121836738dc3ab3d2d3ded5268c35b6 diff --git a/packages/pandas-gbq/.kokoro/requirements.txt b/packages/pandas-gbq/.kokoro/requirements.txt index 385f2d4d6106..d15994bac93c 100644 --- a/packages/pandas-gbq/.kokoro/requirements.txt +++ b/packages/pandas-gbq/.kokoro/requirements.txt @@ -325,31 +325,30 @@ platformdirs==2.5.2 \ --hash=sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788 \ --hash=sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19 # via virtualenv -protobuf==3.20.1 \ - --hash=sha256:06059eb6953ff01e56a25cd02cca1a9649a75a7e65397b5b9b4e929ed71d10cf \ - --hash=sha256:097c5d8a9808302fb0da7e20edf0b8d4703274d140fd25c5edabddcde43e081f \ - --hash=sha256:284f86a6207c897542d7e956eb243a36bb8f9564c1742b253462386e96c6b78f \ - --hash=sha256:32ca378605b41fd180dfe4e14d3226386d8d1b002ab31c969c366549e66a2bb7 \ - --hash=sha256:3cc797c9d15d7689ed507b165cd05913acb992d78b379f6014e013f9ecb20996 \ - --hash=sha256:62f1b5c4cd6c5402b4e2d63804ba49a327e0c386c99b1675c8a0fefda23b2067 \ - --hash=sha256:69ccfdf3657ba59569c64295b7d51325f91af586f8d5793b734260dfe2e94e2c \ - --hash=sha256:6f50601512a3d23625d8a85b1638d914a0970f17920ff39cec63aaef80a93fb7 \ - --hash=sha256:7403941f6d0992d40161aa8bb23e12575637008a5a02283a930addc0508982f9 \ - --hash=sha256:755f3aee41354ae395e104d62119cb223339a8f3276a0cd009ffabfcdd46bb0c \ - --hash=sha256:77053d28427a29987ca9caf7b72ccafee011257561259faba8dd308fda9a8739 \ - --hash=sha256:7e371f10abe57cee5021797126c93479f59fccc9693dafd6bd5633ab67808a91 \ - --hash=sha256:9016d01c91e8e625141d24ec1b20fed584703e527d28512aa8c8707f105a683c \ - --hash=sha256:9be73ad47579abc26c12024239d3540e6b765182a91dbc88e23658ab71767153 \ - --hash=sha256:adc31566d027f45efe3f44eeb5b1f329da43891634d61c75a5944e9be6dd42c9 \ - --hash=sha256:adfc6cf69c7f8c50fd24c793964eef18f0ac321315439d94945820612849c388 \ - --hash=sha256:af0ebadc74e281a517141daad9d0f2c5d93ab78e9d455113719a45a49da9db4e \ - --hash=sha256:cb29edb9eab15742d791e1025dd7b6a8f6fcb53802ad2f6e3adcb102051063ab \ - --hash=sha256:cd68be2559e2a3b84f517fb029ee611546f7812b1fdd0aa2ecc9bc6ec0e4fdde \ - --hash=sha256:cdee09140e1cd184ba9324ec1df410e7147242b94b5f8b0c64fc89e38a8ba531 \ - --hash=sha256:db977c4ca738dd9ce508557d4fce0f5aebd105e158c725beec86feb1f6bc20d8 \ - --hash=sha256:dd5789b2948ca702c17027c84c2accb552fc30f4622a98ab5c51fcfe8c50d3e7 \ - --hash=sha256:e250a42f15bf9d5b09fe1b293bdba2801cd520a9f5ea2d7fb7536d4441811d20 \ - --hash=sha256:ff8d8fa42675249bb456f5db06c00de6c2f4c27a065955917b28c4f15978b9c3 +protobuf==3.20.2 \ + --hash=sha256:03d76b7bd42ac4a6e109742a4edf81ffe26ffd87c5993126d894fe48a120396a \ + --hash=sha256:09e25909c4297d71d97612f04f41cea8fa8510096864f2835ad2f3b3df5a5559 \ + --hash=sha256:18e34a10ae10d458b027d7638a599c964b030c1739ebd035a1dfc0e22baa3bfe \ + --hash=sha256:291fb4307094bf5ccc29f424b42268640e00d5240bf0d9b86bf3079f7576474d \ + --hash=sha256:2c0b040d0b5d5d207936ca2d02f00f765906622c07d3fa19c23a16a8ca71873f \ + --hash=sha256:384164994727f274cc34b8abd41a9e7e0562801361ee77437099ff6dfedd024b \ + --hash=sha256:3cb608e5a0eb61b8e00fe641d9f0282cd0eedb603be372f91f163cbfbca0ded0 \ + --hash=sha256:5d9402bf27d11e37801d1743eada54372f986a372ec9679673bfcc5c60441151 \ + --hash=sha256:712dca319eee507a1e7df3591e639a2b112a2f4a62d40fe7832a16fd19151750 \ + --hash=sha256:7a5037af4e76c975b88c3becdf53922b5ffa3f2cddf657574a4920a3b33b80f3 \ + --hash=sha256:8228e56a865c27163d5d1d1771d94b98194aa6917bcfb6ce139cbfa8e3c27334 \ + --hash=sha256:84a1544252a933ef07bb0b5ef13afe7c36232a774affa673fc3636f7cee1db6c \ + --hash=sha256:84fe5953b18a383fd4495d375fe16e1e55e0a3afe7b4f7b4d01a3a0649fcda9d \ + --hash=sha256:9c673c8bfdf52f903081816b9e0e612186684f4eb4c17eeb729133022d6032e3 \ + --hash=sha256:9f876a69ca55aed879b43c295a328970306e8e80a263ec91cf6e9189243c613b \ + --hash=sha256:a9e5ae5a8e8985c67e8944c23035a0dff2c26b0f5070b2f55b217a1c33bbe8b1 \ + --hash=sha256:b4fdb29c5a7406e3f7ef176b2a7079baa68b5b854f364c21abe327bbeec01cdb \ + --hash=sha256:c184485e0dfba4dfd451c3bd348c2e685d6523543a0f91b9fd4ae90eb09e8422 \ + --hash=sha256:c9cdf251c582c16fd6a9f5e95836c90828d51b0069ad22f463761d27c6c19019 \ + --hash=sha256:e39cf61bb8582bda88cdfebc0db163b774e7e03364bbf9ce1ead13863e81e359 \ + --hash=sha256:e8fbc522303e09036c752a0afcc5c0603e917222d8bedc02813fd73b4b4ed804 \ + --hash=sha256:f34464ab1207114e73bba0794d1257c150a2b89b7a9faf504e00af7c9fd58978 \ + --hash=sha256:f52dabc96ca99ebd2169dadbe018824ebda08a795c7684a0b7d203a290f3adb0 # via # gcp-docuploader # gcp-releasetool diff --git a/packages/pandas-gbq/testing/constraints-3.7.txt b/packages/pandas-gbq/testing/constraints-3.7.txt index 2d9b95f8ae2c..a405afddc0c2 100644 --- a/packages/pandas-gbq/testing/constraints-3.7.txt +++ b/packages/pandas-gbq/testing/constraints-3.7.txt @@ -16,4 +16,4 @@ pandas==0.24.2 pyarrow==3.0.0 pydata-google-auth==0.1.2 tqdm==4.23.0 -protobuf==3.19.0 +protobuf==3.20.2 From 2e4e3e9d92f1ae2db27b8e25e18e7c50364c43e4 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 7 Oct 2022 16:39:47 -0400 Subject: [PATCH 324/519] fix(deps): allow protobuf 3.19.5 (#577) --- packages/pandas-gbq/testing/constraints-3.7.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/testing/constraints-3.7.txt b/packages/pandas-gbq/testing/constraints-3.7.txt index a405afddc0c2..1d9efed73017 100644 --- a/packages/pandas-gbq/testing/constraints-3.7.txt +++ b/packages/pandas-gbq/testing/constraints-3.7.txt @@ -16,4 +16,4 @@ pandas==0.24.2 pyarrow==3.0.0 pydata-google-auth==0.1.2 tqdm==4.23.0 -protobuf==3.20.2 +protobuf==3.19.5 From c8e13cabcbf58d73d3da7378a096b8474424fd1a Mon Sep 17 00:00:00 2001 From: aribray <45905583+aribray@users.noreply.github.com> Date: Mon, 7 Nov 2022 11:36:12 -0600 Subject: [PATCH 325/519] feat: map "if_exists" value to LoadJobConfig.WriteDisposition (#583) * feat: map "if_exists" value to LoadJobConfig.WriteDisposition This uses LoadJobConfig.WriteDisposition to replace if_exists='fail'/'replace'/'append' behavior in to_gbq() ### Dependency updates - Update the minimum version of `db-dtypes` to 1.0.4 - Update the minimum version of `google-api-core` to 2.10.2 - Update the minimum version of `google-auth` to 2.13.0 - Update the minimum version of `google-auth-oauthlib` to 0.7.0 - Update the minimum version of `google-cloud-bigquery` to 3.3.5 - Update the minimum version of `google-cloud-bigquery-storage` to 2.16.2 - Update the minimum version of `pandas` to 1.1.4 - Update the minimum version of `pydata-google-auth` to 1.4.0 --- .../ci/requirements-3.7-0.24.2.conda | 15 +-- packages/pandas-gbq/pandas_gbq/gbq.py | 99 ++++++++----------- packages/pandas-gbq/pandas_gbq/load.py | 18 ++-- packages/pandas-gbq/requirements.txt | 2 +- packages/pandas-gbq/setup.py | 16 +-- .../pandas-gbq/testing/constraints-3.7.txt | 16 +-- packages/pandas-gbq/tests/system/test_gbq.py | 12 ++- packages/pandas-gbq/tests/unit/test_load.py | 19 +++- packages/pandas-gbq/tests/unit/test_to_gbq.py | 36 ++----- 9 files changed, 112 insertions(+), 121 deletions(-) diff --git a/packages/pandas-gbq/ci/requirements-3.7-0.24.2.conda b/packages/pandas-gbq/ci/requirements-3.7-0.24.2.conda index 2facfb2cd44d..2d61383ea2a1 100644 --- a/packages/pandas-gbq/ci/requirements-3.7-0.24.2.conda +++ b/packages/pandas-gbq/ci/requirements-3.7-0.24.2.conda @@ -1,14 +1,17 @@ codecov coverage -db-dtypes==0.3.1 +db-dtypes fastavro flake8 freezegun -numpy==1.16.6 -google-cloud-bigquery==1.27.2 -google-cloud-bigquery-storage==1.1.0 -pyarrow==3.0.0 +numpy +google-api-core +google-auth +google-cloud-bigquery +google-cloud-bigquery-storage +pyarrow pydata-google-auth pytest pytest-cov -tqdm==4.23.0 +requests-oauthlib +tqdm diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 56d6fd706807..820999982823 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -20,10 +20,7 @@ if typing.TYPE_CHECKING: # pragma: NO COVER import pandas -from pandas_gbq.exceptions import ( - AccessDenied, - GenericGBQException, -) +from pandas_gbq.exceptions import AccessDenied, GenericGBQException from pandas_gbq.features import FEATURES import pandas_gbq.schema import pandas_gbq.timestamp @@ -116,20 +113,12 @@ class InvalidSchema(ValueError): table in BigQuery. """ - def __init__( - self, message: str, local_schema: Dict[str, Any], remote_schema: Dict[str, Any] - ): - super().__init__(message) - self._local_schema = local_schema - self._remote_schema = remote_schema - - @property - def local_schema(self) -> Dict[str, Any]: - return self._local_schema + def __init__(self, message: str): + self._message = message @property - def remote_schema(self) -> Dict[str, Any]: - return self._remote_schema + def message(self) -> str: + return self._message class NotFoundException(ValueError): @@ -155,7 +144,12 @@ class TableCreationError(ValueError): Raised when the create table method fails """ - pass + def __init__(self, message: str): + self._message = message + + @property + def message(self) -> str: + return self._message class Context(object): @@ -382,8 +376,14 @@ def process_http_error(ex): if "cancelled" in ex.message: raise QueryTimeout("Reason: {0}".format(ex)) - - raise GenericGBQException("Reason: {0}".format(ex)) + elif "Provided Schema does not match" in ex.message: + error_message = ex.errors[0]["message"] + raise InvalidSchema(f"Reason: {error_message}") + elif "Already Exists: Table" in ex.message: + error_message = ex.errors[0]["message"] + raise TableCreationError(f"Reason: {error_message}") + else: + raise GenericGBQException("Reason: {0}".format(ex)) def download_table( self, @@ -577,6 +577,7 @@ def load_data( self, dataframe, destination_table_ref, + write_disposition, chunksize=None, schema=None, progress_bar=True, @@ -596,6 +597,7 @@ def load_data( schema=schema, location=self.location, api_method=api_method, + write_disposition=write_disposition, billing_project=billing_project, ) if progress_bar and tqdm: @@ -609,11 +611,6 @@ def load_data( except self.http_error as ex: self.process_http_error(ex) - def delete_and_recreate_table(self, project_id, dataset_id, table_id, table_schema): - table = _Table(project_id, dataset_id, credentials=self.credentials) - table.delete(table_id) - table.create(table_id, table_schema) - def _bqschema_to_nullsafe_dtypes(schema_fields): """Specify explicit dtypes based on BigQuery schema. @@ -975,11 +972,9 @@ def to_gbq( ): """Write a DataFrame to a Google BigQuery table. - The main method a user calls to export pandas DataFrame contents to - Google BigQuery table. + The main method a user calls to export pandas DataFrame contents to Google BigQuery table. - This method uses the Google Cloud client library to make requests to - Google BigQuery, documented `here + This method uses the Google Cloud client library to make requests to Google BigQuery, documented `here `__. See the :ref:`How to authenticate with Google BigQuery ` @@ -1114,15 +1109,21 @@ def to_gbq( stacklevel=2, ) - if if_exists not in ("fail", "replace", "append"): - raise ValueError("'{0}' is not valid for if_exists".format(if_exists)) - if "." not in destination_table: raise NotFoundException( "Invalid Table Name. Should be of the form 'datasetId.tableId' or " "'projectId.datasetId.tableId'" ) + if if_exists not in ("fail", "replace", "append"): + raise ValueError("'{0}' is not valid for if_exists".format(if_exists)) + + if_exists_list = ["fail", "replace", "append"] + dispositions = ["WRITE_EMPTY", "WRITE_TRUNCATE", "WRITE_APPEND"] + dispositions_dict = dict(zip(if_exists_list, dispositions)) + + write_disposition = dispositions_dict[if_exists] + connector = GbqConnector( project_id, reauth=reauth, @@ -1142,17 +1143,20 @@ def to_gbq( table_id = destination_table_ref.table_id default_schema = _generate_bq_schema(dataframe) + # If table_schema isn't provided, we'll create one for you if not table_schema: table_schema = default_schema + # It table_schema is provided, we'll update the default_schema to the provided table_schema else: table_schema = pandas_gbq.schema.update_schema( default_schema, dict(fields=table_schema) ) - # If table exists, check if_exists parameter try: + # Try to get the table table = bqclient.get_table(destination_table_ref) except google_exceptions.NotFound: + # If the table doesn't already exist, create it table_connector = _Table( project_id_table, dataset_id, @@ -1161,34 +1165,12 @@ def to_gbq( ) table_connector.create(table_id, table_schema) else: + # Convert original schema (the schema that already exists) to pandas-gbq API format original_schema = pandas_gbq.schema.to_pandas_gbq(table.schema) - if if_exists == "fail": - raise TableCreationError( - "Could not create the table because it " - "already exists. " - "Change the if_exists parameter to " - "'append' or 'replace' data." - ) - elif if_exists == "replace": - connector.delete_and_recreate_table( - project_id_table, dataset_id, table_id, table_schema - ) - else: - if not pandas_gbq.schema.schema_is_subset(original_schema, table_schema): - raise InvalidSchema( - "Please verify that the structure and " - "data types in the DataFrame match the " - "schema of the destination table.", - table_schema, - original_schema, - ) - - # Update the local `table_schema` so mode (NULLABLE/REQUIRED) - # matches. See: https://github.com/pydata/pandas-gbq/issues/315 - table_schema = pandas_gbq.schema.update_schema( - table_schema, original_schema - ) + # Update the local `table_schema` so mode (NULLABLE/REQUIRED) + # matches. See: https://github.com/pydata/pandas-gbq/issues/315 + table_schema = pandas_gbq.schema.update_schema(table_schema, original_schema) if dataframe.empty: # Create the table (if needed), but don't try to run a load job with an @@ -1198,6 +1180,7 @@ def to_gbq( connector.load_data( dataframe, destination_table_ref, + write_disposition=write_disposition, chunksize=chunksize, schema=table_schema, progress_bar=progress_bar, diff --git a/packages/pandas-gbq/pandas_gbq/load.py b/packages/pandas-gbq/pandas_gbq/load.py index 1032806957bf..bad99584284e 100644 --- a/packages/pandas-gbq/pandas_gbq/load.py +++ b/packages/pandas-gbq/pandas_gbq/load.py @@ -113,13 +113,13 @@ def load_parquet( client: bigquery.Client, dataframe: pandas.DataFrame, destination_table_ref: bigquery.TableReference, + write_disposition: str, location: Optional[str], schema: Optional[Dict[str, Any]], billing_project: Optional[str] = None, ): job_config = bigquery.LoadJobConfig() - job_config.write_disposition = "WRITE_APPEND" - job_config.create_disposition = "CREATE_NEVER" + job_config.write_disposition = write_disposition job_config.source_format = "PARQUET" if schema is not None: @@ -143,13 +143,13 @@ def load_parquet( def load_csv( dataframe: pandas.DataFrame, + write_disposition: str, chunksize: Optional[int], bq_schema: Optional[List[bigquery.SchemaField]], load_chunk: Callable, ): job_config = bigquery.LoadJobConfig() - job_config.write_disposition = "WRITE_APPEND" - job_config.create_disposition = "CREATE_NEVER" + job_config.write_disposition = write_disposition job_config.source_format = "CSV" job_config.allow_quoted_newlines = True @@ -167,6 +167,7 @@ def load_csv_from_dataframe( client: bigquery.Client, dataframe: pandas.DataFrame, destination_table_ref: bigquery.TableReference, + write_disposition: str, location: Optional[str], chunksize: Optional[int], schema: Optional[Dict[str, Any]], @@ -187,13 +188,14 @@ def load_chunk(chunk, job_config): project=billing_project, ).result() - return load_csv(dataframe, chunksize, bq_schema, load_chunk) + return load_csv(dataframe, write_disposition, chunksize, bq_schema, load_chunk) def load_csv_from_file( client: bigquery.Client, dataframe: pandas.DataFrame, destination_table_ref: bigquery.TableReference, + write_disposition: str, location: Optional[str], chunksize: Optional[int], schema: Optional[Dict[str, Any]], @@ -223,7 +225,7 @@ def load_chunk(chunk, job_config): finally: chunk_buffer.close() - return load_csv(dataframe, chunksize, bq_schema, load_chunk) + return load_csv(dataframe, write_disposition, chunksize, bq_schema, load_chunk) def load_chunks( @@ -234,6 +236,7 @@ def load_chunks( schema=None, location=None, api_method="load_parquet", + write_disposition="WRITE_EMPTY", billing_project: Optional[str] = None, ): if api_method == "load_parquet": @@ -241,6 +244,7 @@ def load_chunks( client, dataframe, destination_table_ref, + write_disposition, location, schema, billing_project=billing_project, @@ -253,6 +257,7 @@ def load_chunks( client, dataframe, destination_table_ref, + write_disposition, location, chunksize, schema, @@ -263,6 +268,7 @@ def load_chunks( client, dataframe, destination_table_ref, + write_disposition, location, chunksize, schema, diff --git a/packages/pandas-gbq/requirements.txt b/packages/pandas-gbq/requirements.txt index 7b3ede9750ae..bf23435f2648 100644 --- a/packages/pandas-gbq/requirements.txt +++ b/packages/pandas-gbq/requirements.txt @@ -2,4 +2,4 @@ pandas google-auth google-auth-oauthlib google-cloud-bigquery -tqdm +tqdm \ No newline at end of file diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index 0bf0c7b26c7c..12c8443cce68 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -23,24 +23,24 @@ release_status = "Development Status :: 4 - Beta" dependencies = [ "setuptools", - "db-dtypes >=0.3.1,<2.0.0", + "db-dtypes >=1.0.4,<2.0.0", "numpy >=1.16.6", - "pandas >=0.24.2", + "pandas >=1.1.4", "pyarrow >=3.0.0, <10.0dev", - "pydata-google-auth", + "pydata-google-auth >=1.4.0", # Note: google-api-core and google-auth are also included via transitive # dependency on google-cloud-bigquery, but this library also uses them # directly. - "google-api-core >= 1.31.5, <3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0", - "google-auth >=1.25.0", - "google-auth-oauthlib >=0.0.1", + "google-api-core >= 2.10.2, <3.0.0dev", + "google-auth >=2.13.0", + "google-auth-oauthlib >=0.7.0", # Require 1.27.* because it has a fix for out-of-bounds timestamps. See: # https://github.com/googleapis/python-bigquery/pull/209 and # https://github.com/googleapis/python-bigquery-pandas/issues/365 # Exclude 2.4.* because it has a bug where waiting for the query can hang # indefinitely. https://github.com/pydata/pandas-gbq/issues/343 - "google-cloud-bigquery >=1.27.2,<4.0.0dev,!=2.4.*", - "google-cloud-bigquery-storage >=1.1.0,<3.0.0dev", + "google-cloud-bigquery >=3.3.5,<4.0.0dev,!=2.4.*", + "google-cloud-bigquery-storage >=2.16.2,<3.0.0dev", ] extras = { "tqdm": "tqdm>=4.23.0", diff --git a/packages/pandas-gbq/testing/constraints-3.7.txt b/packages/pandas-gbq/testing/constraints-3.7.txt index 1d9efed73017..569287ad835d 100644 --- a/packages/pandas-gbq/testing/constraints-3.7.txt +++ b/packages/pandas-gbq/testing/constraints-3.7.txt @@ -5,15 +5,15 @@ # # e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", # Then this file should have foo==1.14.0 -db-dtypes==0.3.1 -google-api-core==1.31.5 -google-auth==1.25.0 -google-auth-oauthlib==0.0.1 -google-cloud-bigquery==1.27.2 -google-cloud-bigquery-storage==1.1.0 +db-dtypes==1.0.4 +google-api-core==2.10.2 +google-auth==2.13.0 +google-auth-oauthlib==0.7.0 +google-cloud-bigquery==3.3.5 +google-cloud-bigquery-storage==2.16.2 numpy==1.16.6 -pandas==0.24.2 +pandas==1.1.4 pyarrow==3.0.0 -pydata-google-auth==0.1.2 +pydata-google-auth==1.4.0 tqdm==4.23.0 protobuf==3.19.5 diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index ee8190b511f7..5b90e8ba5fb6 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -673,9 +673,17 @@ def test_upload_data_if_table_exists_fail(self, project_id): test_id = "2" test_size = 10 df = make_mixed_dataframe_v2(test_size) - self.table.create(TABLE_ID + test_id, gbq._generate_bq_schema(df)) - # Test the default value of if_exists is 'fail' + # Initialize table with sample data + gbq.to_gbq( + df, + self.destination_table + test_id, + project_id, + chunksize=10000, + credentials=self.credentials, + ) + + # Test the default value of if_exists == 'fail' with pytest.raises(gbq.TableCreationError): gbq.to_gbq( df, diff --git a/packages/pandas-gbq/tests/unit/test_load.py b/packages/pandas-gbq/tests/unit/test_load.py index f2209bdac1f7..1d99d9b4186b 100644 --- a/packages/pandas-gbq/tests/unit/test_load.py +++ b/packages/pandas-gbq/tests/unit/test_load.py @@ -108,7 +108,7 @@ def test_load_csv_from_dataframe_allows_client_to_generate_schema(mock_bigquery_ _ = list( load.load_csv_from_dataframe( - mock_bigquery_client, df, destination, None, None, None + mock_bigquery_client, df, destination, None, None, None, None ) ) @@ -151,7 +151,9 @@ def test_load_csv_from_file_generates_schema(mock_bigquery_client): ) _ = list( - load.load_csv_from_file(mock_bigquery_client, df, destination, None, None, None) + load.load_csv_from_file( + mock_bigquery_client, df, destination, None, None, None, None + ) ) mock_load = mock_bigquery_client.load_table_from_file @@ -222,7 +224,7 @@ def test_load_chunks_omits_policy_tags( def test_load_chunks_with_invalid_api_method(): with pytest.raises(ValueError, match="Got unexpected api_method:"): - load.load_chunks(None, None, None, api_method="not_a_thing") + load.load_chunks(None, None, None, None, api_method="not_a_thing") def test_load_parquet_allows_client_to_generate_schema(mock_bigquery_client): @@ -233,7 +235,14 @@ def test_load_parquet_allows_client_to_generate_schema(mock_bigquery_client): "my-project.my_dataset.my_table" ) - load.load_parquet(mock_bigquery_client, df, destination, None, None) + load.load_parquet( + mock_bigquery_client, + df, + destination, + None, + None, + None, + ) mock_load = mock_bigquery_client.load_table_from_dataframe assert mock_load.called @@ -255,7 +264,7 @@ def test_load_parquet_with_bad_conversion(mock_bigquery_client): ) with pytest.raises(exceptions.ConversionError): - load.load_parquet(mock_bigquery_client, df, destination, None, None) + load.load_parquet(mock_bigquery_client, df, destination, None, None, None) @pytest.mark.parametrize( diff --git a/packages/pandas-gbq/tests/unit/test_to_gbq.py b/packages/pandas-gbq/tests/unit/test_to_gbq.py index c8b419edd111..4456df0ebf10 100644 --- a/packages/pandas-gbq/tests/unit/test_to_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_to_gbq.py @@ -94,7 +94,11 @@ def test_to_gbq_with_if_exists_append_mismatch(mock_bigquery_client): "myproj.my_dataset.my_table", schema=(SchemaField("col_a", "INTEGER"), SchemaField("col_b", "STRING")), ) - with pytest.raises(gbq.InvalidSchema) as exception_block: + mock_bigquery_client.side_effect = gbq.InvalidSchema( + message=r"Provided Schema does not match Table *" + ) + + with pytest.raises((gbq.InvalidSchema)) as exception_block: gbq.to_gbq( DataFrame({"col_a": [0.25, 1.5, -1.0]}), "my_dataset.my_table", @@ -103,16 +107,10 @@ def test_to_gbq_with_if_exists_append_mismatch(mock_bigquery_client): ) exc = exception_block.value - assert exc.remote_schema == { - "fields": [ - {"name": "col_a", "type": "INTEGER", "mode": "NULLABLE"}, - {"name": "col_b", "type": "STRING", "mode": "NULLABLE"}, - ] - } - assert exc.local_schema == {"fields": [{"name": "col_a", "type": "FLOAT"}]} + assert exc.message == r"Provided Schema does not match Table *" -def test_to_gbq_with_if_exists_replace(mock_bigquery_client): +def test_to_gbq_with_if_exists_replace(mock_bigquery_client, expected_load_method): mock_bigquery_client.get_table.side_effect = ( # Initial check google.cloud.bigquery.Table("myproj.my_dataset.my_table"), @@ -125,10 +123,7 @@ def test_to_gbq_with_if_exists_replace(mock_bigquery_client): project_id="myproj", if_exists="replace", ) - # TODO: We can avoid these API calls by using write disposition in the load - # job. See: https://github.com/googleapis/python-bigquery-pandas/issues/118 - assert mock_bigquery_client.delete_table.called - assert mock_bigquery_client.create_table.called + expected_load_method.assert_called_once() def test_to_gbq_with_if_exists_replace_cross_project( @@ -146,20 +141,7 @@ def test_to_gbq_with_if_exists_replace_cross_project( project_id="billing-project", if_exists="replace", ) - # TODO: We can avoid these API calls by using write disposition in the load - # job. See: https://github.com/googleapis/python-bigquery-pandas/issues/118 - assert mock_bigquery_client.delete_table.called - args, _ = mock_bigquery_client.delete_table.call_args - table_delete: google.cloud.bigquery.TableReference = args[0] - assert table_delete.project == "data-project" - assert table_delete.dataset_id == "my_dataset" - assert table_delete.table_id == "my_table" - assert mock_bigquery_client.create_table.called - args, _ = mock_bigquery_client.create_table.call_args - table_create: google.cloud.bigquery.TableReference = args[0] - assert table_create.project == "data-project" - assert table_create.dataset_id == "my_dataset" - assert table_create.table_id == "my_table" + expected_load_method.assert_called_once() # Check that billing project and destination table is set correctly. expected_load_method.assert_called_once() From 7a625fd653019d8b3d215ec6a0176a547d9bf11a Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Sat, 19 Nov 2022 11:34:16 -0500 Subject: [PATCH 326/519] chore(python): update release script dependencies (#590) Source-Link: https://github.com/googleapis/synthtool/commit/25083af347468dd5f90f69627420f7d452b6c50e Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:e6cbd61f1838d9ff6a31436dfc13717f372a7482a82fc1863ca954ec47bff8c8 Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 2 +- .../pandas-gbq/.github/workflows/docs.yml | 4 +- .../pandas-gbq/.github/workflows/lint.yml | 2 +- .../pandas-gbq/.github/workflows/unittest.yml | 2 +- .../pandas-gbq/.kokoro/docker/docs/Dockerfile | 12 +- packages/pandas-gbq/.kokoro/requirements.in | 4 +- packages/pandas-gbq/.kokoro/requirements.txt | 354 ++++++++++-------- packages/pandas-gbq/noxfile.py | 15 +- 8 files changed, 218 insertions(+), 177 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 3815c983cb16..3f1ccc085ef7 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:7a40313731a7cb1454eef6b33d3446ebb121836738dc3ab3d2d3ded5268c35b6 + digest: sha256:e6cbd61f1838d9ff6a31436dfc13717f372a7482a82fc1863ca954ec47bff8c8 diff --git a/packages/pandas-gbq/.github/workflows/docs.yml b/packages/pandas-gbq/.github/workflows/docs.yml index 7092a139aed3..e97d89e484c9 100644 --- a/packages/pandas-gbq/.github/workflows/docs.yml +++ b/packages/pandas-gbq/.github/workflows/docs.yml @@ -12,7 +12,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.9" - name: Install nox run: | python -m pip install --upgrade setuptools pip wheel @@ -28,7 +28,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.9" - name: Install nox run: | python -m pip install --upgrade setuptools pip wheel diff --git a/packages/pandas-gbq/.github/workflows/lint.yml b/packages/pandas-gbq/.github/workflows/lint.yml index d2aee5b7d8ec..16d5a9e90f6d 100644 --- a/packages/pandas-gbq/.github/workflows/lint.yml +++ b/packages/pandas-gbq/.github/workflows/lint.yml @@ -12,7 +12,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.8" - name: Install nox run: | python -m pip install --upgrade setuptools pip wheel diff --git a/packages/pandas-gbq/.github/workflows/unittest.yml b/packages/pandas-gbq/.github/workflows/unittest.yml index 19d1f4d663b7..deffeda1a7bc 100644 --- a/packages/pandas-gbq/.github/workflows/unittest.yml +++ b/packages/pandas-gbq/.github/workflows/unittest.yml @@ -41,7 +41,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.8" - name: Install coverage run: | python -m pip install --upgrade setuptools pip wheel diff --git a/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile b/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile index 238b87b9d1c9..f8137d0ae497 100644 --- a/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile +++ b/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile @@ -60,16 +60,16 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ && rm -f /var/cache/apt/archives/*.deb -###################### Install python 3.8.11 +###################### Install python 3.9.13 -# Download python 3.8.11 -RUN wget https://www.python.org/ftp/python/3.8.11/Python-3.8.11.tgz +# Download python 3.9.13 +RUN wget https://www.python.org/ftp/python/3.9.13/Python-3.9.13.tgz # Extract files -RUN tar -xvf Python-3.8.11.tgz +RUN tar -xvf Python-3.9.13.tgz -# Install python 3.8.11 -RUN ./Python-3.8.11/configure --enable-optimizations +# Install python 3.9.13 +RUN ./Python-3.9.13/configure --enable-optimizations RUN make altinstall ###################### Install pip diff --git a/packages/pandas-gbq/.kokoro/requirements.in b/packages/pandas-gbq/.kokoro/requirements.in index 7718391a34d7..cbd7e77f44db 100644 --- a/packages/pandas-gbq/.kokoro/requirements.in +++ b/packages/pandas-gbq/.kokoro/requirements.in @@ -5,4 +5,6 @@ typing-extensions twine wheel setuptools -nox \ No newline at end of file +nox +charset-normalizer<3 +click<8.1.0 diff --git a/packages/pandas-gbq/.kokoro/requirements.txt b/packages/pandas-gbq/.kokoro/requirements.txt index d15994bac93c..9c1b9be34e6b 100644 --- a/packages/pandas-gbq/.kokoro/requirements.txt +++ b/packages/pandas-gbq/.kokoro/requirements.txt @@ -20,9 +20,9 @@ cachetools==5.2.0 \ --hash=sha256:6a94c6402995a99c3970cc7e4884bb60b4a8639938157eeed436098bf9831757 \ --hash=sha256:f9f17d2aec496a9aa6b76f53e3b614c965223c061982d434d160f930c698a9db # via google-auth -certifi==2022.6.15 \ - --hash=sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d \ - --hash=sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412 +certifi==2022.9.24 \ + --hash=sha256:0d9c601124e5a6ba9712dbc60d9c53c21e34f5f641fe83002317394311bdce14 \ + --hash=sha256:90c1a32f1d68f940488354e36370f6cca89f0f106db09518524c88d6ed83f382 # via requests cffi==1.15.1 \ --hash=sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5 \ @@ -93,11 +93,14 @@ cffi==1.15.1 \ charset-normalizer==2.1.1 \ --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 \ --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f - # via requests + # via + # -r requirements.in + # requests click==8.0.4 \ --hash=sha256:6a7a62563bbfabfda3a38f3023a1db4a35978c0abd76f6c9605ecd6554d6d9b1 \ --hash=sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb # via + # -r requirements.in # gcp-docuploader # gcp-releasetool colorlog==6.7.0 \ @@ -110,29 +113,33 @@ commonmark==0.9.1 \ --hash=sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60 \ --hash=sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9 # via rich -cryptography==37.0.4 \ - --hash=sha256:190f82f3e87033821828f60787cfa42bff98404483577b591429ed99bed39d59 \ - --hash=sha256:2be53f9f5505673eeda5f2736bea736c40f051a739bfae2f92d18aed1eb54596 \ - --hash=sha256:30788e070800fec9bbcf9faa71ea6d8068f5136f60029759fd8c3efec3c9dcb3 \ - --hash=sha256:3d41b965b3380f10e4611dbae366f6dc3cefc7c9ac4e8842a806b9672ae9add5 \ - --hash=sha256:4c590ec31550a724ef893c50f9a97a0c14e9c851c85621c5650d699a7b88f7ab \ - --hash=sha256:549153378611c0cca1042f20fd9c5030d37a72f634c9326e225c9f666d472884 \ - --hash=sha256:63f9c17c0e2474ccbebc9302ce2f07b55b3b3fcb211ded18a42d5764f5c10a82 \ - --hash=sha256:6bc95ed67b6741b2607298f9ea4932ff157e570ef456ef7ff0ef4884a134cc4b \ - --hash=sha256:7099a8d55cd49b737ffc99c17de504f2257e3787e02abe6d1a6d136574873441 \ - --hash=sha256:75976c217f10d48a8b5a8de3d70c454c249e4b91851f6838a4e48b8f41eb71aa \ - --hash=sha256:7bc997818309f56c0038a33b8da5c0bfbb3f1f067f315f9abd6fc07ad359398d \ - --hash=sha256:80f49023dd13ba35f7c34072fa17f604d2f19bf0989f292cedf7ab5770b87a0b \ - --hash=sha256:91ce48d35f4e3d3f1d83e29ef4a9267246e6a3be51864a5b7d2247d5086fa99a \ - --hash=sha256:a958c52505c8adf0d3822703078580d2c0456dd1d27fabfb6f76fe63d2971cd6 \ - --hash=sha256:b62439d7cd1222f3da897e9a9fe53bbf5c104fff4d60893ad1355d4c14a24157 \ - --hash=sha256:b7f8dd0d4c1f21759695c05a5ec8536c12f31611541f8904083f3dc582604280 \ - --hash=sha256:d204833f3c8a33bbe11eda63a54b1aad7aa7456ed769a982f21ec599ba5fa282 \ - --hash=sha256:e007f052ed10cc316df59bc90fbb7ff7950d7e2919c9757fd42a2b8ecf8a5f67 \ - --hash=sha256:f2dcb0b3b63afb6df7fd94ec6fbddac81b5492513f7b0436210d390c14d46ee8 \ - --hash=sha256:f721d1885ecae9078c3f6bbe8a88bc0786b6e749bf32ccec1ef2b18929a05046 \ - --hash=sha256:f7a6de3e98771e183645181b3627e2563dcde3ce94a9e42a3f427d2255190327 \ - --hash=sha256:f8c0a6e9e1dd3eb0414ba320f85da6b0dcbd543126e30fcc546e7372a7fbf3b9 +cryptography==38.0.3 \ + --hash=sha256:068147f32fa662c81aebab95c74679b401b12b57494872886eb5c1139250ec5d \ + --hash=sha256:06fc3cc7b6f6cca87bd56ec80a580c88f1da5306f505876a71c8cfa7050257dd \ + --hash=sha256:25c1d1f19729fb09d42e06b4bf9895212292cb27bb50229f5aa64d039ab29146 \ + --hash=sha256:402852a0aea73833d982cabb6d0c3bb582c15483d29fb7085ef2c42bfa7e38d7 \ + --hash=sha256:4e269dcd9b102c5a3d72be3c45d8ce20377b8076a43cbed6f660a1afe365e436 \ + --hash=sha256:5419a127426084933076132d317911e3c6eb77568a1ce23c3ac1e12d111e61e0 \ + --hash=sha256:554bec92ee7d1e9d10ded2f7e92a5d70c1f74ba9524947c0ba0c850c7b011828 \ + --hash=sha256:5e89468fbd2fcd733b5899333bc54d0d06c80e04cd23d8c6f3e0542358c6060b \ + --hash=sha256:65535bc550b70bd6271984d9863a37741352b4aad6fb1b3344a54e6950249b55 \ + --hash=sha256:6ab9516b85bebe7aa83f309bacc5f44a61eeb90d0b4ec125d2d003ce41932d36 \ + --hash=sha256:6addc3b6d593cd980989261dc1cce38263c76954d758c3c94de51f1e010c9a50 \ + --hash=sha256:728f2694fa743a996d7784a6194da430f197d5c58e2f4e278612b359f455e4a2 \ + --hash=sha256:785e4056b5a8b28f05a533fab69febf5004458e20dad7e2e13a3120d8ecec75a \ + --hash=sha256:78cf5eefac2b52c10398a42765bfa981ce2372cbc0457e6bf9658f41ec3c41d8 \ + --hash=sha256:7f836217000342d448e1c9a342e9163149e45d5b5eca76a30e84503a5a96cab0 \ + --hash=sha256:8d41a46251bf0634e21fac50ffd643216ccecfaf3701a063257fe0b2be1b6548 \ + --hash=sha256:984fe150f350a3c91e84de405fe49e688aa6092b3525f407a18b9646f6612320 \ + --hash=sha256:9b24bcff7853ed18a63cfb0c2b008936a9554af24af2fb146e16d8e1aed75748 \ + --hash=sha256:b1b35d9d3a65542ed2e9d90115dfd16bbc027b3f07ee3304fc83580f26e43249 \ + --hash=sha256:b1b52c9e5f8aa2b802d48bd693190341fae201ea51c7a167d69fc48b60e8a959 \ + --hash=sha256:bbf203f1a814007ce24bd4d51362991d5cb90ba0c177a9c08825f2cc304d871f \ + --hash=sha256:be243c7e2bfcf6cc4cb350c0d5cdf15ca6383bbcb2a8ef51d3c9411a9d4386f0 \ + --hash=sha256:bfbe6ee19615b07a98b1d2287d6a6073f734735b49ee45b11324d85efc4d5cbd \ + --hash=sha256:c46837ea467ed1efea562bbeb543994c2d1f6e800785bd5a2c98bc096f5cb220 \ + --hash=sha256:dfb4f4dd568de1b6af9f4cda334adf7d72cf5bc052516e1b2608b683375dd95c \ + --hash=sha256:ed7b00096790213e09eb11c97cc6e2b757f15f3d2f85833cd2d3ec3fe37c1722 # via # gcp-releasetool # secretstorage @@ -148,23 +155,23 @@ filelock==3.8.0 \ --hash=sha256:55447caa666f2198c5b6b13a26d2084d26fa5b115c00d065664b2124680c4edc \ --hash=sha256:617eb4e5eedc82fc5f47b6d61e4d11cb837c56cb4544e39081099fa17ad109d4 # via virtualenv -gcp-docuploader==0.6.3 \ - --hash=sha256:ba8c9d76b3bbac54b0311c503a373b00edc2dc02d6d54ea9507045adb8e870f7 \ - --hash=sha256:c0f5aaa82ce1854a386197e4e359b120ad6d4e57ae2c812fce42219a3288026b +gcp-docuploader==0.6.4 \ + --hash=sha256:01486419e24633af78fd0167db74a2763974765ee8078ca6eb6964d0ebd388af \ + --hash=sha256:70861190c123d907b3b067da896265ead2eeb9263969d6955c9e0bb091b5ccbf # via -r requirements.in -gcp-releasetool==1.8.7 \ - --hash=sha256:3d2a67c9db39322194afb3b427e9cb0476ce8f2a04033695f0aeb63979fc2b37 \ - --hash=sha256:5e4d28f66e90780d77f3ecf1e9155852b0c3b13cbccb08ab07e66b2357c8da8d +gcp-releasetool==1.10.0 \ + --hash=sha256:72a38ca91b59c24f7e699e9227c90cbe4dd71b789383cb0164b088abae294c83 \ + --hash=sha256:8c7c99320208383d4bb2b808c6880eb7a81424afe7cdba3c8d84b25f4f0e097d # via -r requirements.in -google-api-core==2.8.2 \ - --hash=sha256:06f7244c640322b508b125903bb5701bebabce8832f85aba9335ec00b3d02edc \ - --hash=sha256:93c6a91ccac79079ac6bbf8b74ee75db970cc899278b97d53bc012f35908cf50 +google-api-core==2.10.2 \ + --hash=sha256:10c06f7739fe57781f87523375e8e1a3a4674bf6392cd6131a3222182b971320 \ + --hash=sha256:34f24bd1d5f72a8c4519773d99ca6bf080a6c4e041b4e9f024fe230191dda62e # via # google-cloud-core # google-cloud-storage -google-auth==2.11.0 \ - --hash=sha256:be62acaae38d0049c21ca90f27a23847245c9f161ff54ede13af2cb6afecbac9 \ - --hash=sha256:ed65ecf9f681832298e29328e1ef0a3676e3732b2e56f41532d45f70a22de0fb +google-auth==2.14.1 \ + --hash=sha256:ccaa901f31ad5cbb562615eb8b664b3dd0bf5404a67618e642307f00613eda4d \ + --hash=sha256:f5d8701633bebc12e0deea4df8abd8aff31c28b355360597f7f2ee60f2e4d016 # via # gcp-releasetool # google-api-core @@ -174,76 +181,102 @@ google-cloud-core==2.3.2 \ --hash=sha256:8417acf6466be2fa85123441696c4badda48db314c607cf1e5d543fa8bdc22fe \ --hash=sha256:b9529ee7047fd8d4bf4a2182de619154240df17fbe60ead399078c1ae152af9a # via google-cloud-storage -google-cloud-storage==2.5.0 \ - --hash=sha256:19a26c66c317ce542cea0830b7e787e8dac2588b6bfa4d3fd3b871ba16305ab0 \ - --hash=sha256:382f34b91de2212e3c2e7b40ec079d27ee2e3dbbae99b75b1bcd8c63063ce235 +google-cloud-storage==2.6.0 \ + --hash=sha256:104ca28ae61243b637f2f01455cc8a05e8f15a2a18ced96cb587241cdd3820f5 \ + --hash=sha256:4ad0415ff61abdd8bb2ae81c1f8f7ec7d91a1011613f2db87c614c550f97bfe9 # via gcp-docuploader -google-crc32c==1.3.0 \ - --hash=sha256:04e7c220798a72fd0f08242bc8d7a05986b2a08a0573396187fd32c1dcdd58b3 \ - --hash=sha256:05340b60bf05b574159e9bd940152a47d38af3fb43803ffe71f11d704b7696a6 \ - --hash=sha256:12674a4c3b56b706153a358eaa1018c4137a5a04635b92b4652440d3d7386206 \ - --hash=sha256:127f9cc3ac41b6a859bd9dc4321097b1a4f6aa7fdf71b4f9227b9e3ebffb4422 \ - --hash=sha256:13af315c3a0eec8bb8b8d80b8b128cb3fcd17d7e4edafc39647846345a3f003a \ - --hash=sha256:1926fd8de0acb9d15ee757175ce7242e235482a783cd4ec711cc999fc103c24e \ - --hash=sha256:226f2f9b8e128a6ca6a9af9b9e8384f7b53a801907425c9a292553a3a7218ce0 \ - --hash=sha256:276de6273eb074a35bc598f8efbc00c7869c5cf2e29c90748fccc8c898c244df \ - --hash=sha256:318f73f5484b5671f0c7f5f63741ab020a599504ed81d209b5c7129ee4667407 \ - --hash=sha256:3bbce1be3687bbfebe29abdb7631b83e6b25da3f4e1856a1611eb21854b689ea \ - --hash=sha256:42ae4781333e331a1743445931b08ebdad73e188fd554259e772556fc4937c48 \ - --hash=sha256:58be56ae0529c664cc04a9c76e68bb92b091e0194d6e3c50bea7e0f266f73713 \ - --hash=sha256:5da2c81575cc3ccf05d9830f9e8d3c70954819ca9a63828210498c0774fda1a3 \ - --hash=sha256:6311853aa2bba4064d0c28ca54e7b50c4d48e3de04f6770f6c60ebda1e975267 \ - --hash=sha256:650e2917660e696041ab3dcd7abac160b4121cd9a484c08406f24c5964099829 \ - --hash=sha256:6a4db36f9721fdf391646685ecffa404eb986cbe007a3289499020daf72e88a2 \ - --hash=sha256:779cbf1ce375b96111db98fca913c1f5ec11b1d870e529b1dc7354b2681a8c3a \ - --hash=sha256:7f6fe42536d9dcd3e2ffb9d3053f5d05221ae3bbcefbe472bdf2c71c793e3183 \ - --hash=sha256:891f712ce54e0d631370e1f4997b3f182f3368179198efc30d477c75d1f44942 \ - --hash=sha256:95c68a4b9b7828ba0428f8f7e3109c5d476ca44996ed9a5f8aac6269296e2d59 \ - --hash=sha256:96a8918a78d5d64e07c8ea4ed2bc44354e3f93f46a4866a40e8db934e4c0d74b \ - --hash=sha256:9c3cf890c3c0ecfe1510a452a165431b5831e24160c5fcf2071f0f85ca5a47cd \ - --hash=sha256:9f58099ad7affc0754ae42e6d87443299f15d739b0ce03c76f515153a5cda06c \ - --hash=sha256:a0b9e622c3b2b8d0ce32f77eba617ab0d6768b82836391e4f8f9e2074582bf02 \ - --hash=sha256:a7f9cbea4245ee36190f85fe1814e2d7b1e5f2186381b082f5d59f99b7f11328 \ - --hash=sha256:bab4aebd525218bab4ee615786c4581952eadc16b1ff031813a2fd51f0cc7b08 \ - --hash=sha256:c124b8c8779bf2d35d9b721e52d4adb41c9bfbde45e6a3f25f0820caa9aba73f \ - --hash=sha256:c9da0a39b53d2fab3e5467329ed50e951eb91386e9d0d5b12daf593973c3b168 \ - --hash=sha256:ca60076c388728d3b6ac3846842474f4250c91efbfe5afa872d3ffd69dd4b318 \ - --hash=sha256:cb6994fff247987c66a8a4e550ef374671c2b82e3c0d2115e689d21e511a652d \ - --hash=sha256:d1c1d6236feab51200272d79b3d3e0f12cf2cbb12b208c835b175a21efdb0a73 \ - --hash=sha256:dd7760a88a8d3d705ff562aa93f8445ead54f58fd482e4f9e2bafb7e177375d4 \ - --hash=sha256:dda4d8a3bb0b50f540f6ff4b6033f3a74e8bf0bd5320b70fab2c03e512a62812 \ - --hash=sha256:e0f1ff55dde0ebcfbef027edc21f71c205845585fffe30d4ec4979416613e9b3 \ - --hash=sha256:e7a539b9be7b9c00f11ef16b55486141bc2cdb0c54762f84e3c6fc091917436d \ - --hash=sha256:eb0b14523758e37802f27b7f8cd973f5f3d33be7613952c0df904b68c4842f0e \ - --hash=sha256:ed447680ff21c14aaceb6a9f99a5f639f583ccfe4ce1a5e1d48eb41c3d6b3217 \ - --hash=sha256:f52a4ad2568314ee713715b1e2d79ab55fab11e8b304fd1462ff5cccf4264b3e \ - --hash=sha256:fbd60c6aaa07c31d7754edbc2334aef50601b7f1ada67a96eb1eb57c7c72378f \ - --hash=sha256:fc28e0db232c62ca0c3600884933178f0825c99be4474cdd645e378a10588125 \ - --hash=sha256:fe31de3002e7b08eb20823b3735b97c86c5926dd0581c7710a680b418a8709d4 \ - --hash=sha256:fec221a051150eeddfdfcff162e6db92c65ecf46cb0f7bb1bf812a1520ec026b \ - --hash=sha256:ff71073ebf0e42258a42a0b34f2c09ec384977e7f6808999102eedd5b49920e3 +google-crc32c==1.5.0 \ + --hash=sha256:024894d9d3cfbc5943f8f230e23950cd4906b2fe004c72e29b209420a1e6b05a \ + --hash=sha256:02c65b9817512edc6a4ae7c7e987fea799d2e0ee40c53ec573a692bee24de876 \ + --hash=sha256:02ebb8bf46c13e36998aeaad1de9b48f4caf545e91d14041270d9dca767b780c \ + --hash=sha256:07eb3c611ce363c51a933bf6bd7f8e3878a51d124acfc89452a75120bc436289 \ + --hash=sha256:1034d91442ead5a95b5aaef90dbfaca8633b0247d1e41621d1e9f9db88c36298 \ + --hash=sha256:116a7c3c616dd14a3de8c64a965828b197e5f2d121fedd2f8c5585c547e87b02 \ + --hash=sha256:19e0a019d2c4dcc5e598cd4a4bc7b008546b0358bd322537c74ad47a5386884f \ + --hash=sha256:1c7abdac90433b09bad6c43a43af253e688c9cfc1c86d332aed13f9a7c7f65e2 \ + --hash=sha256:1e986b206dae4476f41bcec1faa057851f3889503a70e1bdb2378d406223994a \ + --hash=sha256:272d3892a1e1a2dbc39cc5cde96834c236d5327e2122d3aaa19f6614531bb6eb \ + --hash=sha256:278d2ed7c16cfc075c91378c4f47924c0625f5fc84b2d50d921b18b7975bd210 \ + --hash=sha256:2ad40e31093a4af319dadf503b2467ccdc8f67c72e4bcba97f8c10cb078207b5 \ + --hash=sha256:2e920d506ec85eb4ba50cd4228c2bec05642894d4c73c59b3a2fe20346bd00ee \ + --hash=sha256:3359fc442a743e870f4588fcf5dcbc1bf929df1fad8fb9905cd94e5edb02e84c \ + --hash=sha256:37933ec6e693e51a5b07505bd05de57eee12f3e8c32b07da7e73669398e6630a \ + --hash=sha256:398af5e3ba9cf768787eef45c803ff9614cc3e22a5b2f7d7ae116df8b11e3314 \ + --hash=sha256:3b747a674c20a67343cb61d43fdd9207ce5da6a99f629c6e2541aa0e89215bcd \ + --hash=sha256:461665ff58895f508e2866824a47bdee72497b091c730071f2b7575d5762ab65 \ + --hash=sha256:4c6fdd4fccbec90cc8a01fc00773fcd5fa28db683c116ee3cb35cd5da9ef6c37 \ + --hash=sha256:5829b792bf5822fd0a6f6eb34c5f81dd074f01d570ed7f36aa101d6fc7a0a6e4 \ + --hash=sha256:596d1f98fc70232fcb6590c439f43b350cb762fb5d61ce7b0e9db4539654cc13 \ + --hash=sha256:5ae44e10a8e3407dbe138984f21e536583f2bba1be9491239f942c2464ac0894 \ + --hash=sha256:635f5d4dd18758a1fbd1049a8e8d2fee4ffed124462d837d1a02a0e009c3ab31 \ + --hash=sha256:64e52e2b3970bd891309c113b54cf0e4384762c934d5ae56e283f9a0afcd953e \ + --hash=sha256:66741ef4ee08ea0b2cc3c86916ab66b6aef03768525627fd6a1b34968b4e3709 \ + --hash=sha256:67b741654b851abafb7bc625b6d1cdd520a379074e64b6a128e3b688c3c04740 \ + --hash=sha256:6ac08d24c1f16bd2bf5eca8eaf8304812f44af5cfe5062006ec676e7e1d50afc \ + --hash=sha256:6f998db4e71b645350b9ac28a2167e6632c239963ca9da411523bb439c5c514d \ + --hash=sha256:72218785ce41b9cfd2fc1d6a017dc1ff7acfc4c17d01053265c41a2c0cc39b8c \ + --hash=sha256:74dea7751d98034887dbd821b7aae3e1d36eda111d6ca36c206c44478035709c \ + --hash=sha256:759ce4851a4bb15ecabae28f4d2e18983c244eddd767f560165563bf9aefbc8d \ + --hash=sha256:77e2fd3057c9d78e225fa0a2160f96b64a824de17840351b26825b0848022906 \ + --hash=sha256:7c074fece789b5034b9b1404a1f8208fc2d4c6ce9decdd16e8220c5a793e6f61 \ + --hash=sha256:7c42c70cd1d362284289c6273adda4c6af8039a8ae12dc451dcd61cdabb8ab57 \ + --hash=sha256:7f57f14606cd1dd0f0de396e1e53824c371e9544a822648cd76c034d209b559c \ + --hash=sha256:83c681c526a3439b5cf94f7420471705bbf96262f49a6fe546a6db5f687a3d4a \ + --hash=sha256:8485b340a6a9e76c62a7dce3c98e5f102c9219f4cfbf896a00cf48caf078d438 \ + --hash=sha256:84e6e8cd997930fc66d5bb4fde61e2b62ba19d62b7abd7a69920406f9ecca946 \ + --hash=sha256:89284716bc6a5a415d4eaa11b1726d2d60a0cd12aadf5439828353662ede9dd7 \ + --hash=sha256:8b87e1a59c38f275c0e3676fc2ab6d59eccecfd460be267ac360cc31f7bcde96 \ + --hash=sha256:8f24ed114432de109aa9fd317278518a5af2d31ac2ea6b952b2f7782b43da091 \ + --hash=sha256:98cb4d057f285bd80d8778ebc4fde6b4d509ac3f331758fb1528b733215443ae \ + --hash=sha256:998679bf62b7fb599d2878aa3ed06b9ce688b8974893e7223c60db155f26bd8d \ + --hash=sha256:9ba053c5f50430a3fcfd36f75aff9caeba0440b2d076afdb79a318d6ca245f88 \ + --hash=sha256:9c99616c853bb585301df6de07ca2cadad344fd1ada6d62bb30aec05219c45d2 \ + --hash=sha256:a1fd716e7a01f8e717490fbe2e431d2905ab8aa598b9b12f8d10abebb36b04dd \ + --hash=sha256:a2355cba1f4ad8b6988a4ca3feed5bff33f6af2d7f134852cf279c2aebfde541 \ + --hash=sha256:b1f8133c9a275df5613a451e73f36c2aea4fe13c5c8997e22cf355ebd7bd0728 \ + --hash=sha256:b8667b48e7a7ef66afba2c81e1094ef526388d35b873966d8a9a447974ed9178 \ + --hash=sha256:ba1eb1843304b1e5537e1fca632fa894d6f6deca8d6389636ee5b4797affb968 \ + --hash=sha256:be82c3c8cfb15b30f36768797a640e800513793d6ae1724aaaafe5bf86f8f346 \ + --hash=sha256:c02ec1c5856179f171e032a31d6f8bf84e5a75c45c33b2e20a3de353b266ebd8 \ + --hash=sha256:c672d99a345849301784604bfeaeba4db0c7aae50b95be04dd651fd2a7310b93 \ + --hash=sha256:c6c777a480337ac14f38564ac88ae82d4cd238bf293f0a22295b66eb89ffced7 \ + --hash=sha256:cae0274952c079886567f3f4f685bcaf5708f0a23a5f5216fdab71f81a6c0273 \ + --hash=sha256:cd67cf24a553339d5062eff51013780a00d6f97a39ca062781d06b3a73b15462 \ + --hash=sha256:d3515f198eaa2f0ed49f8819d5732d70698c3fa37384146079b3799b97667a94 \ + --hash=sha256:d5280312b9af0976231f9e317c20e4a61cd2f9629b7bfea6a693d1878a264ebd \ + --hash=sha256:de06adc872bcd8c2a4e0dc51250e9e65ef2ca91be023b9d13ebd67c2ba552e1e \ + --hash=sha256:e1674e4307fa3024fc897ca774e9c7562c957af85df55efe2988ed9056dc4e57 \ + --hash=sha256:e2096eddb4e7c7bdae4bd69ad364e55e07b8316653234a56552d9c988bd2d61b \ + --hash=sha256:e560628513ed34759456a416bf86b54b2476c59144a9138165c9a1575801d0d9 \ + --hash=sha256:edfedb64740750e1a3b16152620220f51d58ff1b4abceb339ca92e934775c27a \ + --hash=sha256:f13cae8cc389a440def0c8c52057f37359014ccbc9dc1f0827936bcd367c6100 \ + --hash=sha256:f314013e7dcd5cf45ab1945d92e713eec788166262ae8deb2cfacd53def27325 \ + --hash=sha256:f583edb943cf2e09c60441b910d6a20b4d9d626c75a36c8fcac01a6c96c01183 \ + --hash=sha256:fd8536e902db7e365f49e7d9029283403974ccf29b13fc7028b97e2295b33556 \ + --hash=sha256:fe70e325aa68fa4b5edf7d1a4b6f691eb04bbccac0ace68e34820d283b5f80d4 # via google-resumable-media -google-resumable-media==2.3.3 \ - --hash=sha256:27c52620bd364d1c8116eaac4ea2afcbfb81ae9139fb3199652fcac1724bfb6c \ - --hash=sha256:5b52774ea7a829a8cdaa8bd2d4c3d4bc660c91b30857ab2668d0eb830f4ea8c5 +google-resumable-media==2.4.0 \ + --hash=sha256:2aa004c16d295c8f6c33b2b4788ba59d366677c0a25ae7382436cb30f776deaa \ + --hash=sha256:8d5518502f92b9ecc84ac46779bd4f09694ecb3ba38a3e7ca737a86d15cbca1f # via google-cloud-storage -googleapis-common-protos==1.56.4 \ - --hash=sha256:8eb2cbc91b69feaf23e32452a7ae60e791e09967d81d4fcc7fc388182d1bd394 \ - --hash=sha256:c25873c47279387cfdcbdafa36149887901d36202cb645a0e4f29686bf6e4417 +googleapis-common-protos==1.57.0 \ + --hash=sha256:27a849d6205838fb6cc3c1c21cb9800707a661bb21c6ce7fb13e99eb1f8a0c46 \ + --hash=sha256:a9f4a1d7f6d9809657b7f1316a1aa527f6664891531bcfcc13b6696e685f443c # via google-api-core -idna==3.3 \ - --hash=sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff \ - --hash=sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d +idna==3.4 \ + --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 \ + --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2 # via requests -importlib-metadata==4.12.0 \ - --hash=sha256:637245b8bab2b6502fcbc752cc4b7a6f6243bb02b31c5c26156ad103d3d45670 \ - --hash=sha256:7401a975809ea1fdc658c3aa4f78cc2195a0e019c5cbc4c06122884e9ae80c23 +importlib-metadata==5.0.0 \ + --hash=sha256:da31db32b304314d044d3c12c79bd59e307889b287ad12ff387b3500835fc2ab \ + --hash=sha256:ddb0e35065e8938f867ed4928d0ae5bf2a53b7773871bfe6bcc7e4fcdc7dea43 # via # -r requirements.in + # keyring # twine -jaraco-classes==3.2.2 \ - --hash=sha256:6745f113b0b588239ceb49532aa09c3ebb947433ce311ef2f8e3ad64ebb74594 \ - --hash=sha256:e6ef6fd3fcf4579a7a019d87d1e56a883f4e4c35cfe925f86731abc58804e647 +jaraco-classes==3.2.3 \ + --hash=sha256:2353de3288bc6b82120752201c6b1c1a14b058267fa424ed5ce5984e3b922158 \ + --hash=sha256:89559fa5c1d3c34eff6f631ad80bb21f378dbcbb35dd161fd2c6b93f5be2f98a # via keyring jeepney==0.8.0 \ --hash=sha256:5efe48d255973902f6badc3ce55e2aa6c5c3b3bc642059ef3a91247bcfcc5806 \ @@ -255,9 +288,9 @@ jinja2==3.1.2 \ --hash=sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852 \ --hash=sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61 # via gcp-releasetool -keyring==23.9.0 \ - --hash=sha256:4c32a31174faaee48f43a7e2c7e9c3216ec5e95acf22a2bebfb4a1d05056ee44 \ - --hash=sha256:98f060ec95ada2ab910c195a2d4317be6ef87936a766b239c46aa3c7aac4f0db +keyring==23.11.0 \ + --hash=sha256:3dd30011d555f1345dec2c262f0153f2f0ca6bca041fb1dc4588349bb4c0ac1e \ + --hash=sha256:ad192263e2cdd5f12875dedc2da13534359a7e760e77f8d04b50968a821c2361 # via # gcp-releasetool # twine @@ -303,9 +336,9 @@ markupsafe==2.1.1 \ --hash=sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a \ --hash=sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7 # via jinja2 -more-itertools==8.14.0 \ - --hash=sha256:1bc4f91ee5b1b31ac7ceacc17c09befe6a40a503907baf9c839c229b5095cfd2 \ - --hash=sha256:c09443cd3d5438b8dafccd867a6bc1cb0894389e90cb53d227456b0b0bccb750 +more-itertools==9.0.0 \ + --hash=sha256:250e83d7e81d0c87ca6bd942e6aeab8cc9daa6096d12c5308f3f92fa5e5c1f41 \ + --hash=sha256:5a6257e40878ef0520b1803990e3e22303a41b5714006c32a3fd8304b26ea1ab # via jaraco-classes nox==2022.8.7 \ --hash=sha256:1b894940551dc5c389f9271d197ca5d655d40bdc6ccf93ed6880e4042760a34b \ @@ -321,34 +354,33 @@ pkginfo==1.8.3 \ --hash=sha256:848865108ec99d4901b2f7e84058b6e7660aae8ae10164e015a6dcf5b242a594 \ --hash=sha256:a84da4318dd86f870a9447a8c98340aa06216bfc6f2b7bdc4b8766984ae1867c # via twine -platformdirs==2.5.2 \ - --hash=sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788 \ - --hash=sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19 +platformdirs==2.5.4 \ + --hash=sha256:1006647646d80f16130f052404c6b901e80ee4ed6bef6792e1f238a8969106f7 \ + --hash=sha256:af0276409f9a02373d540bf8480021a048711d572745aef4b7842dad245eba10 # via virtualenv -protobuf==3.20.2 \ - --hash=sha256:03d76b7bd42ac4a6e109742a4edf81ffe26ffd87c5993126d894fe48a120396a \ - --hash=sha256:09e25909c4297d71d97612f04f41cea8fa8510096864f2835ad2f3b3df5a5559 \ - --hash=sha256:18e34a10ae10d458b027d7638a599c964b030c1739ebd035a1dfc0e22baa3bfe \ - --hash=sha256:291fb4307094bf5ccc29f424b42268640e00d5240bf0d9b86bf3079f7576474d \ - --hash=sha256:2c0b040d0b5d5d207936ca2d02f00f765906622c07d3fa19c23a16a8ca71873f \ - --hash=sha256:384164994727f274cc34b8abd41a9e7e0562801361ee77437099ff6dfedd024b \ - --hash=sha256:3cb608e5a0eb61b8e00fe641d9f0282cd0eedb603be372f91f163cbfbca0ded0 \ - --hash=sha256:5d9402bf27d11e37801d1743eada54372f986a372ec9679673bfcc5c60441151 \ - --hash=sha256:712dca319eee507a1e7df3591e639a2b112a2f4a62d40fe7832a16fd19151750 \ - --hash=sha256:7a5037af4e76c975b88c3becdf53922b5ffa3f2cddf657574a4920a3b33b80f3 \ - --hash=sha256:8228e56a865c27163d5d1d1771d94b98194aa6917bcfb6ce139cbfa8e3c27334 \ - --hash=sha256:84a1544252a933ef07bb0b5ef13afe7c36232a774affa673fc3636f7cee1db6c \ - --hash=sha256:84fe5953b18a383fd4495d375fe16e1e55e0a3afe7b4f7b4d01a3a0649fcda9d \ - --hash=sha256:9c673c8bfdf52f903081816b9e0e612186684f4eb4c17eeb729133022d6032e3 \ - --hash=sha256:9f876a69ca55aed879b43c295a328970306e8e80a263ec91cf6e9189243c613b \ - --hash=sha256:a9e5ae5a8e8985c67e8944c23035a0dff2c26b0f5070b2f55b217a1c33bbe8b1 \ - --hash=sha256:b4fdb29c5a7406e3f7ef176b2a7079baa68b5b854f364c21abe327bbeec01cdb \ - --hash=sha256:c184485e0dfba4dfd451c3bd348c2e685d6523543a0f91b9fd4ae90eb09e8422 \ - --hash=sha256:c9cdf251c582c16fd6a9f5e95836c90828d51b0069ad22f463761d27c6c19019 \ - --hash=sha256:e39cf61bb8582bda88cdfebc0db163b774e7e03364bbf9ce1ead13863e81e359 \ - --hash=sha256:e8fbc522303e09036c752a0afcc5c0603e917222d8bedc02813fd73b4b4ed804 \ - --hash=sha256:f34464ab1207114e73bba0794d1257c150a2b89b7a9faf504e00af7c9fd58978 \ - --hash=sha256:f52dabc96ca99ebd2169dadbe018824ebda08a795c7684a0b7d203a290f3adb0 +protobuf==3.20.3 \ + --hash=sha256:03038ac1cfbc41aa21f6afcbcd357281d7521b4157926f30ebecc8d4ea59dcb7 \ + --hash=sha256:28545383d61f55b57cf4df63eebd9827754fd2dc25f80c5253f9184235db242c \ + --hash=sha256:2e3427429c9cffebf259491be0af70189607f365c2f41c7c3764af6f337105f2 \ + --hash=sha256:398a9e0c3eaceb34ec1aee71894ca3299605fa8e761544934378bbc6c97de23b \ + --hash=sha256:44246bab5dd4b7fbd3c0c80b6f16686808fab0e4aca819ade6e8d294a29c7050 \ + --hash=sha256:447d43819997825d4e71bf5769d869b968ce96848b6479397e29fc24c4a5dfe9 \ + --hash=sha256:67a3598f0a2dcbc58d02dd1928544e7d88f764b47d4a286202913f0b2801c2e7 \ + --hash=sha256:74480f79a023f90dc6e18febbf7b8bac7508420f2006fabd512013c0c238f454 \ + --hash=sha256:819559cafa1a373b7096a482b504ae8a857c89593cf3a25af743ac9ecbd23480 \ + --hash=sha256:899dc660cd599d7352d6f10d83c95df430a38b410c1b66b407a6b29265d66469 \ + --hash=sha256:8c0c984a1b8fef4086329ff8dd19ac77576b384079247c770f29cc8ce3afa06c \ + --hash=sha256:9aae4406ea63d825636cc11ffb34ad3379335803216ee3a856787bcf5ccc751e \ + --hash=sha256:a7ca6d488aa8ff7f329d4c545b2dbad8ac31464f1d8b1c87ad1346717731e4db \ + --hash=sha256:b6cc7ba72a8850621bfec987cb72623e703b7fe2b9127a161ce61e61558ad905 \ + --hash=sha256:bf01b5720be110540be4286e791db73f84a2b721072a3711efff6c324cdf074b \ + --hash=sha256:c02ce36ec760252242a33967d51c289fd0e1c0e6e5cc9397e2279177716add86 \ + --hash=sha256:d9e4432ff660d67d775c66ac42a67cf2453c27cb4d738fc22cb53b5d84c135d4 \ + --hash=sha256:daa564862dd0d39c00f8086f88700fdbe8bc717e993a21e90711acfed02f2402 \ + --hash=sha256:de78575669dddf6099a8a0f46a27e82a1783c557ccc38ee620ed8cc96d3be7d7 \ + --hash=sha256:e64857f395505ebf3d2569935506ae0dfc4a15cb80dc25261176c784662cdcc4 \ + --hash=sha256:f4bd856d702e5b0d96a00ec6b307b0f51c1982c2bf9c0052cf9019e9a544ba99 \ + --hash=sha256:f4c42102bc82a51108e449cbb32b19b180022941c727bac0cfd50170341f16ee # via # gcp-docuploader # gcp-releasetool @@ -377,9 +409,9 @@ pygments==2.13.0 \ # via # readme-renderer # rich -pyjwt==2.4.0 \ - --hash=sha256:72d1d253f32dbd4f5c88eaf1fdc62f3a19f676ccbadb9dbc5d07e951b2b26daf \ - --hash=sha256:d42908208c699b3b973cbeb01a969ba6a96c821eefb1c5bfe4c390c01d67abba +pyjwt==2.6.0 \ + --hash=sha256:69285c7e31fc44f68a1feb309e948e0df53259d579295e6cfe2b1792329f05fd \ + --hash=sha256:d83c3d892a77bbb74d3e1a2cfa90afaadb60945205d1095d9221f04466f64c14 # via gcp-releasetool pyparsing==3.0.9 \ --hash=sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb \ @@ -392,9 +424,9 @@ python-dateutil==2.8.2 \ --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 # via gcp-releasetool -readme-renderer==37.0 \ - --hash=sha256:07b7ea234e03e58f77cc222e206e6abb8f4c0435becce5104794ee591f9301c5 \ - --hash=sha256:9fa416704703e509eeb900696751c908ddeb2011319d93700d8f18baff887a69 +readme-renderer==37.3 \ + --hash=sha256:cd653186dfc73055656f090f227f5cb22a046d7f71a841dfa305f55c9a513273 \ + --hash=sha256:f67a16caedfa71eef48a31b39708637a6f4664c4394801a7b0d6432d13907343 # via twine requests==2.28.1 \ --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983 \ @@ -405,17 +437,17 @@ requests==2.28.1 \ # google-cloud-storage # requests-toolbelt # twine -requests-toolbelt==0.9.1 \ - --hash=sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f \ - --hash=sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0 +requests-toolbelt==0.10.1 \ + --hash=sha256:18565aa58116d9951ac39baa288d3adb5b3ff975c4f25eee78555d89e8f247f7 \ + --hash=sha256:62e09f7ff5ccbda92772a29f394a49c3ad6cb181d568b1337626b2abb628a63d # via twine rfc3986==2.0.0 \ --hash=sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd \ --hash=sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c # via twine -rich==12.5.1 \ - --hash=sha256:2eb4e6894cde1e017976d2975ac210ef515d7548bc595ba20e195fb9628acdeb \ - --hash=sha256:63a5c5ce3673d3d5fbbf23cd87e11ab84b6b451436f1b7f19ec54b6bc36ed7ca +rich==12.6.0 \ + --hash=sha256:a4eb26484f2c82589bd9a17c73d32a010b1e29d89f1604cd9bf3a2097b81bb5e \ + --hash=sha256:ba3a3775974105c221d31141f2c116f4fd65c5ceb0698657a11e9f295ec93fd0 # via twine rsa==4.9 \ --hash=sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7 \ @@ -437,9 +469,9 @@ twine==4.0.1 \ --hash=sha256:42026c18e394eac3e06693ee52010baa5313e4811d5a11050e7d48436cf41b9e \ --hash=sha256:96b1cf12f7ae611a4a40b6ae8e9570215daff0611828f5fe1f37a16255ab24a0 # via -r requirements.in -typing-extensions==4.3.0 \ - --hash=sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02 \ - --hash=sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6 +typing-extensions==4.4.0 \ + --hash=sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa \ + --hash=sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e # via -r requirements.in urllib3==1.26.12 \ --hash=sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e \ @@ -447,25 +479,25 @@ urllib3==1.26.12 \ # via # requests # twine -virtualenv==20.16.4 \ - --hash=sha256:014f766e4134d0008dcaa1f95bafa0fb0f575795d07cae50b1bee514185d6782 \ - --hash=sha256:035ed57acce4ac35c82c9d8802202b0e71adac011a511ff650cbcf9635006a22 +virtualenv==20.16.7 \ + --hash=sha256:8691e3ff9387f743e00f6bb20f70121f5e4f596cae754531f2b3b3a1b1ac696e \ + --hash=sha256:efd66b00386fdb7dbe4822d172303f40cd05e50e01740b19ea42425cbe653e29 # via nox webencodings==0.5.1 \ --hash=sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78 \ --hash=sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923 # via bleach -wheel==0.37.1 \ - --hash=sha256:4bdcd7d840138086126cd09254dc6195fb4fc6f01c050a1d7236f2630db1d22a \ - --hash=sha256:e9a504e793efbca1b8e0e9cb979a249cf4a0a7b5b8c9e8b65a5e39d49529c1c4 +wheel==0.38.4 \ + --hash=sha256:965f5259b566725405b05e7cf774052044b1ed30119b5d586b2703aafe8719ac \ + --hash=sha256:b60533f3f5d530e971d6737ca6d58681ee434818fab630c83a734bb10c083ce8 # via -r requirements.in -zipp==3.8.1 \ - --hash=sha256:05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2 \ - --hash=sha256:47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009 +zipp==3.10.0 \ + --hash=sha256:4fcb6f278987a6605757302a6e40e896257570d11c51628968ccb2a47e80c6c1 \ + --hash=sha256:7a7262fd930bd3e36c50b9a64897aec3fafff3dfdeec9623ae22b40e93f99bb8 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -setuptools==65.2.0 \ - --hash=sha256:7f4bc85450898a09f76ebf28b72fa25bc7111f6c7d665d514a60bba9c75ef2a9 \ - --hash=sha256:a3ca5857c89f82f5c9410e8508cb32f4872a3bafd4aa7ae122a24ca33bccc750 +setuptools==65.5.1 \ + --hash=sha256:d0b9a8433464d5800cbe05094acf5c6d52a91bfac9b52bcfc4d41382be5d5d31 \ + --hash=sha256:e197a19aa8ec9722928f2206f8de752def0e4c9fc6953527360d1c36d94ddb2f # via -r requirements.in diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 07b7345f093f..c79af71333f0 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -364,12 +364,16 @@ def cover(session): session.run("coverage", "erase") -@nox.session(python=DEFAULT_PYTHON_VERSION) +@nox.session(python="3.9") def docs(session): """Build the docs for this library.""" session.install("-e", ".") - session.install("sphinx==4.0.1", "alabaster", "recommonmark") + session.install( + "sphinx==4.0.1", + "alabaster", + "recommonmark", + ) shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) session.run( @@ -386,13 +390,16 @@ def docs(session): ) -@nox.session(python=DEFAULT_PYTHON_VERSION) +@nox.session(python="3.9") def docfx(session): """Build the docfx yaml files for this library.""" session.install("-e", ".") session.install( - "sphinx==4.0.1", "alabaster", "recommonmark", "gcp-sphinx-docfx-yaml" + "sphinx==4.0.1", + "alabaster", + "recommonmark", + "gcp-sphinx-docfx-yaml", ) shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) From 8f4753a0274a9acd0b0c27629680e5e2017be1aa Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 23 Nov 2022 18:48:02 -0800 Subject: [PATCH 327/519] chore(main): release 0.18.0 (#586) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- packages/pandas-gbq/CHANGELOG.md | 7 +++++++ packages/pandas-gbq/pandas_gbq/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index c1865ef6fdba..321b64cf73e2 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.18.0](https://github.com/googleapis/python-bigquery-pandas/compare/v0.17.9...v0.18.0) (2022-11-19) + + +### Features + +* Map "if_exists" value to LoadJobConfig.WriteDisposition ([#583](https://github.com/googleapis/python-bigquery-pandas/issues/583)) ([7389cd2](https://github.com/googleapis/python-bigquery-pandas/commit/7389cd2a363ebf403b66905ca845ca842a754922)) + ## [0.17.9](https://github.com/googleapis/python-bigquery-pandas/compare/v0.17.8...v0.17.9) (2022-09-27) diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py index 8614d84c6b0b..2da1dafa2995 100644 --- a/packages/pandas-gbq/pandas_gbq/version.py +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -2,4 +2,4 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -__version__ = "0.17.9" +__version__ = "0.18.0" From d834a5ee41920ed870fdc85bfc56573b9491710e Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Sat, 26 Nov 2022 19:01:22 -0500 Subject: [PATCH 328/519] chore(python): drop flake8-import-order in samples noxfile (#591) Source-Link: https://github.com/googleapis/synthtool/commit/6ed3a831cb9ff69ef8a504c353e098ec0192ad93 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:3abfa0f1886adaf0b83f07cb117b24a639ea1cb9cffe56d43280b977033563eb Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 2 +- .../pandas-gbq/samples/snippets/noxfile.py | 26 +++---------------- 2 files changed, 4 insertions(+), 24 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 3f1ccc085ef7..bb21147e4c23 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:e6cbd61f1838d9ff6a31436dfc13717f372a7482a82fc1863ca954ec47bff8c8 + digest: sha256:3abfa0f1886adaf0b83f07cb117b24a639ea1cb9cffe56d43280b977033563eb diff --git a/packages/pandas-gbq/samples/snippets/noxfile.py b/packages/pandas-gbq/samples/snippets/noxfile.py index b053ca568f63..e8283c38d4a0 100644 --- a/packages/pandas-gbq/samples/snippets/noxfile.py +++ b/packages/pandas-gbq/samples/snippets/noxfile.py @@ -18,7 +18,7 @@ import os from pathlib import Path import sys -from typing import Callable, Dict, List, Optional +from typing import Callable, Dict, Optional import nox @@ -109,22 +109,6 @@ def get_pytest_env_vars() -> Dict[str, str]: # -def _determine_local_import_names(start_dir: str) -> List[str]: - """Determines all import names that should be considered "local". - - This is used when running the linter to insure that import order is - properly checked. - """ - file_ext_pairs = [os.path.splitext(path) for path in os.listdir(start_dir)] - return [ - basename - for basename, extension in file_ext_pairs - if extension == ".py" - or os.path.isdir(os.path.join(start_dir, basename)) - and basename not in ("__pycache__") - ] - - # Linting with flake8. # # We ignore the following rules: @@ -139,7 +123,6 @@ def _determine_local_import_names(start_dir: str) -> List[str]: "--show-source", "--builtin=gettext", "--max-complexity=20", - "--import-order-style=google", "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py", "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202", "--max-line-length=88", @@ -149,14 +132,11 @@ def _determine_local_import_names(start_dir: str) -> List[str]: @nox.session def lint(session: nox.sessions.Session) -> None: if not TEST_CONFIG["enforce_type_hints"]: - session.install("flake8", "flake8-import-order") + session.install("flake8") else: - session.install("flake8", "flake8-import-order", "flake8-annotations") + session.install("flake8", "flake8-annotations") - local_names = _determine_local_import_names(".") args = FLAKE8_COMMON_ARGS + [ - "--application-import-names", - ",".join(local_names), ".", ] session.run("flake8", *args) From 9d95e7e299ce55885f196bf16bf4acf4bd08ad68 Mon Sep 17 00:00:00 2001 From: EugeneTorap Date: Mon, 28 Nov 2022 23:56:13 +0300 Subject: [PATCH 329/519] deps: Remove upper bound for python and pyarrow (#592) Remove upper bound for pyarrow in order to use pyarrow 10.0.1 which supports python 3.11 --- packages/pandas-gbq/setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index 12c8443cce68..225768c93881 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -26,7 +26,7 @@ "db-dtypes >=1.0.4,<2.0.0", "numpy >=1.16.6", "pandas >=1.1.4", - "pyarrow >=3.0.0, <10.0dev", + "pyarrow >=3.0.0", "pydata-google-auth >=1.4.0", # Note: google-api-core and google-auth are also included via transitive # dependency on google-cloud-bigquery, but this library also uses them @@ -89,6 +89,7 @@ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Operating System :: OS Independent", "Topic :: Internet", "Topic :: Scientific/Engineering", @@ -97,7 +98,7 @@ packages=packages, install_requires=dependencies, extras_require=extras, - python_requires=">=3.7, <3.11", + python_requires=">=3.7", include_package_data=True, zip_safe=False, ) From 741ba93ab24265caad684e0e04f35ecfd9df093e Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 29 Nov 2022 09:47:18 -0600 Subject: [PATCH 330/519] chore(main): release 0.18.1 (#593) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- packages/pandas-gbq/CHANGELOG.md | 7 +++++++ packages/pandas-gbq/pandas_gbq/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index 321b64cf73e2..7409fa7a5251 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.18.1](https://github.com/googleapis/python-bigquery-pandas/compare/v0.18.0...v0.18.1) (2022-11-28) + + +### Dependencies + +* Remove upper bound for python and pyarrow ([#592](https://github.com/googleapis/python-bigquery-pandas/issues/592)) ([4d28684](https://github.com/googleapis/python-bigquery-pandas/commit/4d286840feaa8290f4d674dce1c269e7e09980d5)) + ## [0.18.0](https://github.com/googleapis/python-bigquery-pandas/compare/v0.17.9...v0.18.0) (2022-11-19) diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py index 2da1dafa2995..53c1d2b4b27e 100644 --- a/packages/pandas-gbq/pandas_gbq/version.py +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -2,4 +2,4 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -__version__ = "0.18.0" +__version__ = "0.18.1" From 5e98c2a14a34fb9948e1fcee8509e9e2a570bb37 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 8 Dec 2022 14:34:29 -0500 Subject: [PATCH 331/519] build(deps): bump certifi from 2022.9.24 to 2022.12.7 in /synthtool/gcp/templates/python_library/.kokoro (#594) Source-Link: https://github.com/googleapis/synthtool/commit/b4fe62efb5114b6738ad4b13d6f654f2bf4b7cc0 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:3bf87e47c2173d7eed42714589dc4da2c07c3268610f1e47f8e1a30decbfc7f1 Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 2 +- packages/pandas-gbq/.kokoro/requirements.txt | 6 +++--- packages/pandas-gbq/.pre-commit-config.yaml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index bb21147e4c23..fccaa8e84449 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:3abfa0f1886adaf0b83f07cb117b24a639ea1cb9cffe56d43280b977033563eb + digest: sha256:3bf87e47c2173d7eed42714589dc4da2c07c3268610f1e47f8e1a30decbfc7f1 diff --git a/packages/pandas-gbq/.kokoro/requirements.txt b/packages/pandas-gbq/.kokoro/requirements.txt index 9c1b9be34e6b..05dc4672edaa 100644 --- a/packages/pandas-gbq/.kokoro/requirements.txt +++ b/packages/pandas-gbq/.kokoro/requirements.txt @@ -20,9 +20,9 @@ cachetools==5.2.0 \ --hash=sha256:6a94c6402995a99c3970cc7e4884bb60b4a8639938157eeed436098bf9831757 \ --hash=sha256:f9f17d2aec496a9aa6b76f53e3b614c965223c061982d434d160f930c698a9db # via google-auth -certifi==2022.9.24 \ - --hash=sha256:0d9c601124e5a6ba9712dbc60d9c53c21e34f5f641fe83002317394311bdce14 \ - --hash=sha256:90c1a32f1d68f940488354e36370f6cca89f0f106db09518524c88d6ed83f382 +certifi==2022.12.7 \ + --hash=sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3 \ + --hash=sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18 # via requests cffi==1.15.1 \ --hash=sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5 \ diff --git a/packages/pandas-gbq/.pre-commit-config.yaml b/packages/pandas-gbq/.pre-commit-config.yaml index 46d237160f6d..5405cc8ff1f3 100644 --- a/packages/pandas-gbq/.pre-commit-config.yaml +++ b/packages/pandas-gbq/.pre-commit-config.yaml @@ -25,7 +25,7 @@ repos: rev: 22.3.0 hooks: - id: black -- repo: https://gitlab.com/pycqa/flake8 +- repo: https://github.com/pycqa/flake8 rev: 3.9.2 hooks: - id: flake8 From 51bd0c449da3961a4d89ec3e9606109294d9e36f Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Wed, 4 Jan 2023 14:47:43 -0500 Subject: [PATCH 332/519] chore: updates python version to 3.11 (#597) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: updates python version to 3.11 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/workflows/unittest.yml | 2 +- packages/pandas-gbq/CONTRIBUTING.rst | 10 ++++++---- packages/pandas-gbq/noxfile.py | 4 ++-- packages/pandas-gbq/owlbot.py | 4 ++-- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/packages/pandas-gbq/.github/workflows/unittest.yml b/packages/pandas-gbq/.github/workflows/unittest.yml index deffeda1a7bc..0de0f6918c27 100644 --- a/packages/pandas-gbq/.github/workflows/unittest.yml +++ b/packages/pandas-gbq/.github/workflows/unittest.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python: ['3.7', '3.8', '3.9', '3.10'] + python: ['3.7', '3.8', '3.9', '3.10', '3.11'] steps: - name: Checkout uses: actions/checkout@v3 diff --git a/packages/pandas-gbq/CONTRIBUTING.rst b/packages/pandas-gbq/CONTRIBUTING.rst index 90bd84f2e34b..b93f22405720 100644 --- a/packages/pandas-gbq/CONTRIBUTING.rst +++ b/packages/pandas-gbq/CONTRIBUTING.rst @@ -22,7 +22,7 @@ In order to add a feature: documentation. - The feature must work fully on the following CPython versions: - 3.7, 3.8, 3.9 and 3.10 on both UNIX and Windows. + 3.7, 3.8, 3.9, 3.10 and 3.11 on both UNIX and Windows. - The feature must not add unnecessary dependencies (where "unnecessary" is of course subjective, but new dependencies should @@ -72,7 +72,7 @@ We use `nox `__ to instrument our tests. - To run a single unit test:: - $ nox -s unit-3.10 -- -k + $ nox -s unit-3.11 -- -k .. note:: @@ -143,12 +143,12 @@ Running System Tests $ nox -s system # Run a single system test - $ nox -s system-3.10 -- -k + $ nox -s system-3.11 -- -k .. note:: - System tests are only configured to run under Python 3.7, 3.8, 3.9 and 3.10. + System tests are only configured to run under Python 3.7, 3.8, 3.9, 3.10 and 3.11. For expediency, we do not run them in older versions of Python 3. This alone will not run the tests. You'll need to change some local @@ -225,11 +225,13 @@ We support: - `Python 3.8`_ - `Python 3.9`_ - `Python 3.10`_ +- `Python 3.11`_ .. _Python 3.7: https://docs.python.org/3.7/ .. _Python 3.8: https://docs.python.org/3.8/ .. _Python 3.9: https://docs.python.org/3.9/ .. _Python 3.10: https://docs.python.org/3.10/ +.. _Python 3.11: https://docs.python.org/3.11/ Supported versions can be found in our ``noxfile.py`` `config`_. diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index c79af71333f0..a4104780cc69 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -31,7 +31,7 @@ DEFAULT_PYTHON_VERSION = "3.8" -UNIT_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] +UNIT_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11"] UNIT_TEST_STANDARD_DEPENDENCIES = [ "mock", "asyncmock", @@ -51,7 +51,7 @@ "3.9": [], } -SYSTEM_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] +SYSTEM_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11"] SYSTEM_TEST_STANDARD_DEPENDENCIES = [ "mock", "pytest", diff --git a/packages/pandas-gbq/owlbot.py b/packages/pandas-gbq/owlbot.py index f5c5c6891440..9c9454f8852f 100644 --- a/packages/pandas-gbq/owlbot.py +++ b/packages/pandas-gbq/owlbot.py @@ -35,8 +35,8 @@ } extras = ["tqdm"] templated_files = common.py_library( - unit_test_python_versions=["3.7", "3.8", "3.9", "3.10"], - system_test_python_versions=["3.7", "3.8", "3.9", "3.10"], + unit_test_python_versions=["3.7", "3.8", "3.9", "3.10", "3.11"], + system_test_python_versions=["3.7", "3.8", "3.9", "3.10", "3.11"], cov_level=96, unit_test_external_dependencies=["freezegun"], unit_test_extras=extras, From 4039dd792ee357569ad690374eb91c2cbbcf1676 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 6 Jan 2023 13:34:31 -0500 Subject: [PATCH 333/519] chore(python): add support for python 3.11 (#599) Source-Link: https://github.com/googleapis/synthtool/commit/7197a001ffb6d8ce7b0b9b11c280f0c536c1033a Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:c43f1d918bcf817d337aa29ff833439494a158a0831508fda4ec75dc4c0d0320 Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 +- .../.kokoro/samples/python3.11/common.cfg | 40 +++++++++++++++++++ .../.kokoro/samples/python3.11/continuous.cfg | 6 +++ .../samples/python3.11/periodic-head.cfg | 11 +++++ .../.kokoro/samples/python3.11/periodic.cfg | 6 +++ .../.kokoro/samples/python3.11/presubmit.cfg | 6 +++ .../pandas-gbq/samples/snippets/noxfile.py | 2 +- 7 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.11/common.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.11/continuous.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.11/periodic-head.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.11/periodic.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.11/presubmit.cfg diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index fccaa8e84449..889f77dfa25d 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -1,4 +1,4 @@ -# Copyright 2022 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:3bf87e47c2173d7eed42714589dc4da2c07c3268610f1e47f8e1a30decbfc7f1 + digest: sha256:c43f1d918bcf817d337aa29ff833439494a158a0831508fda4ec75dc4c0d0320 diff --git a/packages/pandas-gbq/.kokoro/samples/python3.11/common.cfg b/packages/pandas-gbq/.kokoro/samples/python3.11/common.cfg new file mode 100644 index 000000000000..b9a38d5d8e26 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.11/common.cfg @@ -0,0 +1,40 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.11" +} + +# Declare build specific Cloud project. +env_vars: { + key: "BUILD_SPECIFIC_GCLOUD_PROJECT" + value: "python-docs-samples-tests-311" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-pandas/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-bigquery-pandas/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.11/continuous.cfg b/packages/pandas-gbq/.kokoro/samples/python3.11/continuous.cfg new file mode 100644 index 000000000000..a1c8d9759c88 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.11/continuous.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.11/periodic-head.cfg b/packages/pandas-gbq/.kokoro/samples/python3.11/periodic-head.cfg new file mode 100644 index 000000000000..98efde4dc99d --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.11/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-pandas/.kokoro/test-samples-against-head.sh" +} diff --git a/packages/pandas-gbq/.kokoro/samples/python3.11/periodic.cfg b/packages/pandas-gbq/.kokoro/samples/python3.11/periodic.cfg new file mode 100644 index 000000000000..71cd1e597e38 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.11/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} diff --git a/packages/pandas-gbq/.kokoro/samples/python3.11/presubmit.cfg b/packages/pandas-gbq/.kokoro/samples/python3.11/presubmit.cfg new file mode 100644 index 000000000000..a1c8d9759c88 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.11/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/packages/pandas-gbq/samples/snippets/noxfile.py b/packages/pandas-gbq/samples/snippets/noxfile.py index e8283c38d4a0..1224cbe212e4 100644 --- a/packages/pandas-gbq/samples/snippets/noxfile.py +++ b/packages/pandas-gbq/samples/snippets/noxfile.py @@ -89,7 +89,7 @@ def get_pytest_env_vars() -> Dict[str, str]: # DO NOT EDIT - automatically generated. # All versions used to test samples. -ALL_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] +ALL_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11"] # Any default versions that should be ignored. IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] From bdabc325c9f3199852284eae3c152d807a7b80a1 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Wed, 11 Jan 2023 12:48:40 -0500 Subject: [PATCH 334/519] feat: adds ability to provide redirect uri (#595) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: adds ability to provide redirect uri * renames redirect_uri variable * adds js folder to sphinx and auth_redirect_uri * feat: adds javascript functionality related to oauth * feat: adds javascript functionality related to oauth * feat: adds sample authcodescript * feat: adds ability to host oauth page with the documentation * adds clarity to the user interface messaging * updates linting, toctree * updates minimum version of pydata-google-auth * Update docs/_static/js/authcodescripts.js * adds check to avoid override of user given clientid, clientsecret * adds parameters to the func to_gbq * adds docstrings related to three new parameters * Apply suggestions from code review * fix rst formatting * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Tim Swast Co-authored-by: Owl Bot --- .../docs/_static/js/authcodescripts.js | 37 +++++++++++++++++ packages/pandas-gbq/docs/index.rst | 1 + packages/pandas-gbq/docs/oauth.rst | 39 ++++++++++++++++++ packages/pandas-gbq/pandas_gbq/auth.py | 19 +++++++-- packages/pandas-gbq/pandas_gbq/gbq.py | 40 +++++++++++++++++++ packages/pandas-gbq/setup.py | 2 +- .../pandas-gbq/testing/constraints-3.7.txt | 2 +- 7 files changed, 135 insertions(+), 5 deletions(-) create mode 100644 packages/pandas-gbq/docs/_static/js/authcodescripts.js create mode 100644 packages/pandas-gbq/docs/oauth.rst diff --git a/packages/pandas-gbq/docs/_static/js/authcodescripts.js b/packages/pandas-gbq/docs/_static/js/authcodescripts.js new file mode 100644 index 000000000000..1cf844b5061a --- /dev/null +++ b/packages/pandas-gbq/docs/_static/js/authcodescripts.js @@ -0,0 +1,37 @@ + + +// Copyright (c) 2023 pandas-gbq Authors All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + + +function onloadoauthcode() { + const PARAMS = new Proxy(new URLSearchParams(window.location.search), { + get: (searchParams, prop) => searchParams.get(prop), + }); + const AUTH_CODE = PARAMS.code; + + document.querySelector('.auth-code').textContent = AUTH_CODE; + + setupCopyButton(document.querySelector('.copy'), AUTH_CODE); +} + +function setupCopyButton(button, text) { + button.addEventListener('click', () => { + navigator.clipboard.writeText(text); + button.textContent = "Verification Code Copied"; + setTimeout(() => { + // Remove the aria-live label so that when the + // button text changes back to "Copy", it is + // not read out. + button.removeAttribute("aria-live"); + button.textContent = "Copy"; + }, 1000); + + // Re-Add the aria-live attribute to enable speech for + // when button text changes next time. + setTimeout(() => { + button.setAttribute("aria-live", "assertive"); + }, 2000); + }); +} \ No newline at end of file diff --git a/packages/pandas-gbq/docs/index.rst b/packages/pandas-gbq/docs/index.rst index e104127d9cd4..5862dc17cd48 100644 --- a/packages/pandas-gbq/docs/index.rst +++ b/packages/pandas-gbq/docs/index.rst @@ -41,6 +41,7 @@ Contents: contributing.rst changelog.md privacy.rst + oauth.rst Indices and tables diff --git a/packages/pandas-gbq/docs/oauth.rst b/packages/pandas-gbq/docs/oauth.rst new file mode 100644 index 000000000000..9d633a4f8668 --- /dev/null +++ b/packages/pandas-gbq/docs/oauth.rst @@ -0,0 +1,39 @@ +.. image:: https://lh3.googleusercontent.com/KaU6SyiIpDKe4tyGPgt7yzGVTsfMqBvP9bL24o_4M58puYDO-nY8-BazrNk3RyhRFJA + :alt: gcloud CLI logo + :class: logo + +Sign in to BigQuery +=================== + +You are seeing this page because you are attempting to access BigQuery via one +of several possible methods, including: + + * the ``pandas_gbq`` library (https://github.com/googleapis/python-bigquery-pandas) + + OR a ``pandas`` library helper function such as: + + * ``pandas.DataFrame.to_gbq()`` + * ``pandas.read_gbq()`` + +from this or another machine. If this is not the case, close this tab. + +Enter the following verification code in the CommandLine Interface (CLI) on the +machine you want to log into. This is a credential **similar to your password** +and should not be shared with others. + + +.. raw:: html + + + +
+ +
+
+ + +.. hint:: + + You can close this tab when you’re done. diff --git a/packages/pandas-gbq/pandas_gbq/auth.py b/packages/pandas-gbq/pandas_gbq/auth.py index 47f989b517f9..0dbc2a07414b 100644 --- a/packages/pandas-gbq/pandas_gbq/auth.py +++ b/packages/pandas-gbq/pandas_gbq/auth.py @@ -28,7 +28,13 @@ def get_credentials( - private_key=None, project_id=None, reauth=False, auth_local_webserver=True + private_key=None, + project_id=None, + reauth=False, + auth_local_webserver=True, + auth_redirect_uri=None, + client_id=None, + client_secret=None, ): import pydata_google_auth @@ -41,12 +47,19 @@ def get_credentials( method from the google-auth package.""" ) + if client_id is None: + client_id = CLIENT_ID + + if client_secret is None: + client_secret = CLIENT_SECRET + credentials, default_project_id = pydata_google_auth.default( SCOPES, - client_id=CLIENT_ID, - client_secret=CLIENT_SECRET, + client_id=client_id, + client_secret=client_secret, credentials_cache=get_credentials_cache(reauth), auth_local_webserver=auth_local_webserver, + redirect_uri=auth_redirect_uri, ) project_id = project_id or default_project_id diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 820999982823..c059a943c11d 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -280,6 +280,9 @@ def __init__( location=None, credentials=None, use_bqstorage_api=False, + auth_redirect_uri=None, + client_id=None, + client_secret=None, ): global context from google.api_core.exceptions import GoogleAPIError @@ -294,6 +297,10 @@ def __init__( self.auth_local_webserver = auth_local_webserver self.dialect = dialect self.credentials = credentials + self.auth_redirect_uri = auth_redirect_uri + self.client_id = client_id + self.client_secret = client_secret + default_project = None # Service account credentials have a project associated with them. @@ -313,6 +320,9 @@ def __init__( project_id=project_id, reauth=reauth, auth_local_webserver=auth_local_webserver, + auth_redirect_uri=auth_redirect_uri, + client_id=client_id, + client_secret=client_secret, ) if self.project_id is None: @@ -735,6 +745,9 @@ def read_gbq( private_key=None, progress_bar_type="tqdm", dtypes=None, + auth_redirect_uri=None, + client_id=None, + client_secret=None, ): r"""Load data from Google BigQuery using google-cloud-python @@ -864,6 +877,15 @@ def read_gbq( or :func:`google.oauth2.service_account.Credentials.from_service_account_file` instead. + auth_redirect_uri : str + Path to the authentication page for organization-specific authentication + workflows. Used when ``auth_local_webserver=False``. + client_id : str + The Client ID for the Google Cloud Project the user is attempting to + connect to. + client_secret : str + The Client Secret associated with the Client ID for the Google Cloud Project + the user is attempting to connect to. Returns ------- @@ -912,6 +934,9 @@ def read_gbq( credentials=credentials, private_key=private_key, use_bqstorage_api=use_bqstorage_api, + auth_redirect_uri=auth_redirect_uri, + client_id=client_id, + client_secret=client_secret, ) if _is_query(query_or_table): @@ -969,6 +994,9 @@ def to_gbq( api_method: str = "default", verbose=None, private_key=None, + auth_redirect_uri=None, + client_id=None, + client_secret=None, ): """Write a DataFrame to a Google BigQuery table. @@ -1071,6 +1099,15 @@ def to_gbq( or :func:`google.oauth2.service_account.Credentials.from_service_account_file` instead. + auth_redirect_uri : str + Path to the authentication page for organization-specific authentication + workflows. Used when ``auth_local_webserver=False``. + client_id : str + The Client ID for the Google Cloud Project the user is attempting to + connect to. + client_secret : str + The Client Secret associated with the Client ID for the Google Cloud Project + the user is attempting to connect to. """ _test_google_api_imports() @@ -1131,6 +1168,9 @@ def to_gbq( location=location, credentials=credentials, private_key=private_key, + auth_redirect_uri=auth_redirect_uri, + client_id=client_id, + client_secret=client_secret, ) bqclient = connector.client diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index 225768c93881..b94a21dc3675 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -27,7 +27,7 @@ "numpy >=1.16.6", "pandas >=1.1.4", "pyarrow >=3.0.0", - "pydata-google-auth >=1.4.0", + "pydata-google-auth >=1.5.0", # Note: google-api-core and google-auth are also included via transitive # dependency on google-cloud-bigquery, but this library also uses them # directly. diff --git a/packages/pandas-gbq/testing/constraints-3.7.txt b/packages/pandas-gbq/testing/constraints-3.7.txt index 569287ad835d..dc26cdee16dc 100644 --- a/packages/pandas-gbq/testing/constraints-3.7.txt +++ b/packages/pandas-gbq/testing/constraints-3.7.txt @@ -14,6 +14,6 @@ google-cloud-bigquery-storage==2.16.2 numpy==1.16.6 pandas==1.1.4 pyarrow==3.0.0 -pydata-google-auth==1.4.0 +pydata-google-auth==1.5.0 tqdm==4.23.0 protobuf==3.19.5 From ae250d3a6015041828d4f879feeb5e3be4c7a941 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 11 Jan 2023 12:38:13 -0600 Subject: [PATCH 335/519] chore(main): release 0.19.0 (#600) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- packages/pandas-gbq/CHANGELOG.md | 7 +++++++ packages/pandas-gbq/pandas_gbq/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index 7409fa7a5251..06598f6fa1f4 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.19.0](https://github.com/googleapis/python-bigquery-pandas/compare/v0.18.1...v0.19.0) (2023-01-11) + + +### Features + +* Adds ability to provide redirect uri ([#595](https://github.com/googleapis/python-bigquery-pandas/issues/595)) ([a06085e](https://github.com/googleapis/python-bigquery-pandas/commit/a06085eddddd18394f623d0b6f3a65d91f51e979)) + ## [0.18.1](https://github.com/googleapis/python-bigquery-pandas/compare/v0.18.0...v0.18.1) (2022-11-28) diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py index 53c1d2b4b27e..96d40a55019a 100644 --- a/packages/pandas-gbq/pandas_gbq/version.py +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -2,4 +2,4 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -__version__ = "0.18.1" +__version__ = "0.19.0" From 597e4d615f71f842d1d39b93470afc3ee255c39a Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Wed, 25 Jan 2023 12:46:51 -0500 Subject: [PATCH 336/519] feat: updates the user instructions re OAuth (#603) * updates the user instructions re OAuth * feat: updates the user instructions re OAuth * splits the content for highly constrained environments vs default environments * update toctree references --- .../pandas-gbq/docs/howto/authentication.rst | 26 ++ ...cation_highly_constrained_environments.rst | 301 ++++++++++++++++++ 2 files changed, 327 insertions(+) create mode 100644 packages/pandas-gbq/docs/howto/authentication_highly_constrained_environments.rst diff --git a/packages/pandas-gbq/docs/howto/authentication.rst b/packages/pandas-gbq/docs/howto/authentication.rst index 029d5f38a5c7..c65563f8d44d 100644 --- a/packages/pandas-gbq/docs/howto/authentication.rst +++ b/packages/pandas-gbq/docs/howto/authentication.rst @@ -1,6 +1,10 @@ Authentication ============== +.. contents:: Table of Contents + :local: + :depth: 1 + Before you begin, you must create a Google Cloud Platform project. Use the `BigQuery sandbox `__ to try the service for free. @@ -190,3 +194,25 @@ credentials are not found. Additional information on the user credentials authentication mechanism can be found in the `Google Cloud authentication guide `__. + +Authenticating from Highly Constrained Development Environments +--------------------------------------------------------------- + +The instructions above may not be adequate for users who are working in +a *highly constrained development environment*: + +Highly constrained development environments typically prevent users from using +the `Default Authentication Methods` and are generally characterized by one or +more of the following circumstances: + +* There are limitations on what you can install on the development environment + (i.e. you can't install ``gcloud``). +* You don't have access to a graphical user interface (i.e. you are remotely + SSH'ed into a headless server and don't have access to a browser to complete + the authentication process used in the default login workflow) . +* The code is being executed in a typical data science context: using a Jupyter + (or similar) notebook. + +If the conditions above apply to you, your needs may be better served +by the content in the `Authentication (Highly Constrained Development Environment) +`_ section. diff --git a/packages/pandas-gbq/docs/howto/authentication_highly_constrained_environments.rst b/packages/pandas-gbq/docs/howto/authentication_highly_constrained_environments.rst new file mode 100644 index 000000000000..a8c100d8de97 --- /dev/null +++ b/packages/pandas-gbq/docs/howto/authentication_highly_constrained_environments.rst @@ -0,0 +1,301 @@ +:orphan: + +Authentication (Highly Constrained Development Environments) +============================================================ + +Before you begin, you must create a Google Cloud Platform project. Use the +`BigQuery sandbox `__ to try +the service for free. + +pandas-gbq `authenticates with the Google BigQuery service +`_ via OAuth 2.0. Use +the ``credentials`` argument to explicitly pass in Google +:class:`~google.auth.credentials.Credentials`. + +.. _authentication_hce: + + +Authenticating from Highly Constrained Development Environments +--------------------------------------------------------------- + +These instructions are primarily for users who are working in a *highly +constrained development environment*. + +Highly constrained development environments typically prevent users from using +the `Default Authentication Methods` and are generally characterized by one or +more of the following circumstances: + +* There are limitations on what you can install on the development environment + (i.e. you can't install ``gcloud``). +* You don't have access to a graphical user interface (i.e. you are remotely + SSH'ed into a headless server and don't have access to a browser to complete + the authentication process used in the default login workflow) . +* The code is being executed in a typical data science context: using a Jupyter + (or similar) notebook. + +If the conditions above **do not** apply to you, your needs may be better served +by the content in the `Default Authentication Methods `_ section. + +When dealing with highly constrained environments, there are two primary options +that one can choose from: Testing Mode OR an institution-specific authentication +page. + +#. Testing Mode: This approach requires that you enable Testing Mode on your + Cloud Project and that you have fewer than 100 users. +#. Institution-specific authentication page: In cases where the Testing Mode + option is not possible and/or there are specific institutional needs, + you/your institution can create and host an institution-specific OAuth + authentication page and associate a redirect URI to that Cloud Project. + +OPTION 1 - Testing Mode +^^^^^^^^^^^^^^^^^^^^^^^ + +This approach is for limited use, such as when testing your product. It is not +intended for production use. If you have fewer than 100 users, it is possible to +configure User Type as External and the Publishing Status of your Project as +Testing Mode to enable OAuth Out-of-Band (OOB) Authentication. NOTE: general +purpose `OOB Authentication was deprecated `_ for all use cases except Testing Mode. + +.. note:: Projects configured with a Publishing Status of Testing are **limited to + up to 100 test users** who must be individually listed in the OAuth consent + screen. A test user consumes a Project's test user quota once added to the + Project. + + Authentications by a test user **will expire seven days from the time of consent.** If your OAuth client requests an offline access type and receives a refresh token, that token will also expire. + + To move a project from Testing Mode to In Production requires app verification + and requires your institution to switch to using an alternate authentication + method, such as an institution-specific authentication page. + +The test users must be manually and individually added to your Cloud Project (i.e. you can not provide a group email alias for your development team because the system does not support alias expansion). + +Google displays a warning message before allowing a specified test user to authenticate scopes requested by your Project's OAuth clients. The warning message confirms the user has test access to your Project and reminds them that they should consider the risks associated with granting access to their data to an unverified app. + +For additional limitations and details about Testing Mode, see: `Setting up your OAuth consent screen `_. + +To enable Testing Mode and add users to your Cloud Project, in your Project dashboard: + +#. Click on **APIs & Services > OAuth consent screen**. +#. Select **External** to enable Testing Mode. +#. Click **Create**. +#. Fill in the necessary details related to the following: + + #. App information + #. App domain, including Authorized domains + #. Developer contact information + +#. Click **Save and Continue**. +#. Click **Add or Remove Scopes** to choose appropriate Scopes for your Project. An *Update selected scopes* dialogue will open. +#. Click **Update**. +#. Click **Save and Continue**. +#. Click **Add Users** to add users to your Project. An *Add Users* dialogue will open. +#. Enter the user's name in the text field. +#. Click **Add**. +#. Click **Save and Continue**. +#. A summary screen will display with all the information you entered. + +To access BigQuery programmatically, you will need your Client ID and your Client Secret, which can be generated as follows: + +#. Click **APIs and Services > Credentials**. +#. Click **+ Create Credentials > OAuth client ID**. +#. Select Desktop app In the **Application Type** field. +#. Fill in the name of your OAuth 2.0 client in the **Name** field. +#. Click **Create**. + +Your Client ID and Client Secret are displayed in the pop-up. There is also a reminder that only test users that are listed on the Oauth consent screen can access the application. The client ID and Client Secret can also be found here, if they have already been generated: + +#. Click on **APIs and Services > Credentials**. +#. Click on the name of your OAuth 2.0 Client under **OAuth 2.0 Client IDs**. +#. The Client ID and Client Secret will be displayed. + +With the Client ID and Client Secret, you are ready to create an OAuth workflow using code similar to the following: + +To run this code sample, you will need to have ``python-bigquery-pandas`` installed. The following dependencies will be installed by ``python-bigquery-pandas``: + +* pydata-google-auth +* google-auth +* google-auth-oauthlib +* pandas +* google-cloud-bigquery +* tqdm + +**Sample code:** ``oauth-read-from-bq-testing-mode.py`` + +.. code:: python + + import pandas_gbq + + projectid = "your-project-name here" + + CLIENT_ID = "your-client-id here" + + # WARNING: for the purposes of this demo code, the Client Secret is + # included here. In your script, take precautions to ensure + # that your Client Secret does not get pushed to a public + # repository or otherwise get compromised + CLIENT_SECRET = "your-client-secret here" + + df = pandas_gbq.read_gbq( + "SELECT SESSION_USER() as user_id, CURRENT_TIMESTAMP() as time", + project_id=projectid, + auth_local_webserver=False, + client_id=CLIENT_ID, + client_secret=CLIENT_SECRET, + ) + + print(df) + +OPTION 2 - Institution-specific authentication page +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To access Bigquery programmatically, you will need your Client ID and your Client Secret, an OAuth authorization page, and an assigned redirect URI. + +To add a Client ID, Client Secret, and Redirect URI to your Cloud Project, in your Project dashboard: + +#. Click on **APIs & Services > OAuth consent screen**. +#. Select **Internal**. +#. Click **Create**. +#. Fill in the necessary details related to the following: + + #. App information + #. App domain, including Authorized domains + #. Developer contact information + +#. Click **Save and Continue**. +#. Click **Add or Remove Scopes** to choose appropriate Scopes for your Project. An Update selected scopes dialogue will open. +#. Click **Update**. +#. Click **Save and Continue**. +#. Click on **APIs and Services > Credentials**. +#. Click on **+ Create Credentials > OAuth client ID**. +#. Select Web application in the **Application Type** field. +#. Fill in the name of your OAuth 2.0 client in the **Name** field. +#. Click **Add Uri** under the Authorized Redirect URIs section. +#. Add a URI for your application (i.e. the path to where you are hosting a file such as the ``oauth.html`` file shown below). +#. Click **Create**. + +Your Client ID and Client Secret will be displayed in the pop-up. The client ID and Client Secret can also be found here: + +#. Click on **APIs and Services > Credentials**. +#. Click on the name of your OAuth 2.0 Client under **OAuth 2.0 Client IDs**. +#. The Client ID and Client Secret and the Authorized Redirect URIs will be displayed. + +You will need to host a webpage (such as ``oauth.html``) with some associated javascript (such as shown below in ``authcodescripts.js``) to parse the results of the OAuth workflow. + +**Code Sample**: ``oauth.html`` + +.. code:: html + + + + + + + +

Sign in to BigQuery

+

You are seeing this page because you are attempting to access BigQuery via one + of several possible methods, including:

+
+
+
    +
  • the pandas_gbq library (https://github.com/googleapis/python-bigquery-pandas)

  • +
+

OR a pandas library helper function such as:

+
    +
  • pandas.DataFrame.to_gbq()

  • +
  • pandas.read_gbq()

  • +
+
+
+

from this or another machine. If this is not the case, close this tab.

+

Enter the following verification code in the CommandLine Interface (CLI) on the + machine you want to log into. This is a credential similar to your password + and should not be shared with others.

+ +
+ +
+
+ + + + +**Code Sample**: ``authcodescripts.js`` + +.. code:: javascript + + function onloadoauthcode() { + const PARAMS = new Proxy(new URLSearchParams(window.location.search), { + get: (searchParams, prop) => searchParams.get(prop), + }); + const AUTH_CODE = PARAMS.code; + document.querySelector('.auth-code').textContent = AUTH_CODE; + setupCopyButton(document.querySelector('.copy'), AUTH_CODE); + } + + function setupCopyButton(button, text) { + button.addEventListener('click', () => { + navigator.clipboard.writeText(text); + button.textContent = "Verification Code Copied"; + setTimeout(() => { + // Remove the aria-live label so that when the + // button text changes back to "Copy", it is + // not read out. + button.removeAttribute("aria-live"); + button.textContent = "Copy"; + }, 1000); + + // Re-Add the aria-live attribute to enable speech for + // when button text changes next time. + setTimeout(() => { + button.setAttribute("aria-live", "assertive"); + }, 2000); + }); + } + +With these items in place: + +* Client ID +* Client Secret +* redirect URI +* authentication page + +you are ready to create an OAuth workflow using code similar to the following: + +To run this code sample, you will need to have ``python-bigquery-pandas`` installed. The following dependencies will be installed by ``python-bigquery-pandas``: + +* pydata-google-auth +* google-auth +* google-auth-oauthlib +* pandas +* google-cloud-bigquery +* tqdm + +**Sample Code**: ``oauth-read-from-bq-org-specific.py`` + +.. code:: python + + import pandas_gbq + + projectid = "your-project-name-here" + + REDIRECT_URI = "your-redirect-uri here/oauth.html" + CLIENT_ID = "your-client-id here" + + # WARNING: for the purposes of this demo code, the Client Secret is + # included here. In your script, take precautions to ensure + # that your Client Secret does not get pushed to a public + # repository or otherwise compromised + CLIENT_SECRET = "your-client-secret here" + + df = pandas_gbq.read_gbq( + "SELECT SESSION_USER() as user_id, CURRENT_TIMESTAMP() as time", + project_id=projectid, + auth_local_webserver=False, + auth_redirect_uri=REDIRECT_URI, + client_id=CLIENT_ID, + client_secret=CLIENT_SECRET, + ) + + print(df) From 9562d80a48d76fab3926651aaf34859b5de3a002 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 30 Jan 2023 09:19:45 -0500 Subject: [PATCH 337/519] chore(main): release 0.19.1 (#606) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- packages/pandas-gbq/CHANGELOG.md | 7 +++++++ packages/pandas-gbq/pandas_gbq/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index 06598f6fa1f4..2deaddba0885 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.19.1](https://github.com/googleapis/python-bigquery-pandas/compare/v0.19.0...v0.19.1) (2023-01-25) + + +### Documentation + +* Updates the user instructions re OAuth ([0c2b716](https://github.com/googleapis/python-bigquery-pandas/commit/0c2b716a55551f26050ed8d94ff912d406527ac5)) + ## [0.19.0](https://github.com/googleapis/python-bigquery-pandas/compare/v0.18.1...v0.19.0) (2023-01-11) diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py index 96d40a55019a..a20f538cfecd 100644 --- a/packages/pandas-gbq/pandas_gbq/version.py +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -2,4 +2,4 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -__version__ = "0.19.0" +__version__ = "0.19.1" From dc37ac20d20936e474e54a061303ac84db366ecb Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Mon, 30 Jan 2023 16:40:19 +0000 Subject: [PATCH 338/519] chore: fix prerelease_deps nox session [autoapprove] (#609) Source-Link: https://togithub.com/googleapis/synthtool/commit/26c7505b2f76981ec1707b851e1595c8c06e90fc Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:f946c75373c2b0040e8e318c5e85d0cf46bc6e61d0a01f3ef94d8de974ac6790 --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 2 +- packages/pandas-gbq/noxfile.py | 14 ++++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 889f77dfa25d..f0f3b24b20cd 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:c43f1d918bcf817d337aa29ff833439494a158a0831508fda4ec75dc4c0d0320 + digest: sha256:f946c75373c2b0040e8e318c5e85d0cf46bc6e61d0a01f3ef94d8de974ac6790 diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index a4104780cc69..e18072ca0ec1 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -196,9 +196,9 @@ def unit(session): def install_systemtest_dependencies(session, *constraints): # Use pre-release gRPC for system tests. - # Exclude version 1.49.0rc1 which has a known issue. - # See https://github.com/grpc/grpc/pull/30642 - session.install("--pre", "grpcio!=1.49.0rc1") + # Exclude version 1.52.0rc1 which has a known issue. + # See https://github.com/grpc/grpc/issues/32163 + session.install("--pre", "grpcio!=1.52.0rc1") session.install(*SYSTEM_TEST_STANDARD_DEPENDENCIES, *constraints) @@ -437,9 +437,7 @@ def prerelease_deps(session): unit_deps_all = UNIT_TEST_STANDARD_DEPENDENCIES + UNIT_TEST_EXTERNAL_DEPENDENCIES session.install(*unit_deps_all) system_deps_all = ( - SYSTEM_TEST_STANDARD_DEPENDENCIES - + SYSTEM_TEST_EXTERNAL_DEPENDENCIES - + SYSTEM_TEST_EXTRAS + SYSTEM_TEST_STANDARD_DEPENDENCIES + SYSTEM_TEST_EXTERNAL_DEPENDENCIES ) session.install(*system_deps_all) @@ -469,8 +467,8 @@ def prerelease_deps(session): # dependency of grpc "six", "googleapis-common-protos", - # Exclude version 1.49.0rc1 which has a known issue. See https://github.com/grpc/grpc/pull/30642 - "grpcio!=1.49.0rc1", + # Exclude version 1.52.0rc1 which has a known issue. See https://github.com/grpc/grpc/issues/32163 + "grpcio!=1.52.0rc1", "grpcio-status", "google-api-core", "proto-plus", From 3369a88fe4c2112d18179082aa41713b3e3f0c30 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Wed, 8 Feb 2023 15:14:12 +0000 Subject: [PATCH 339/519] build(deps): bump cryptography from 38.0.3 to 39.0.1 in /synthtool/gcp/templates/python_library/.kokoro (#612) Source-Link: https://togithub.com/googleapis/synthtool/commit/bb171351c3946d3c3c32e60f5f18cee8c464ec51 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:f62c53736eccb0c4934a3ea9316e0d57696bb49c1a7c86c726e9bb8a2f87dadf --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 2 +- packages/pandas-gbq/.kokoro/requirements.txt | 49 +++++++++---------- 2 files changed, 23 insertions(+), 28 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index f0f3b24b20cd..894fb6bc9b47 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:f946c75373c2b0040e8e318c5e85d0cf46bc6e61d0a01f3ef94d8de974ac6790 + digest: sha256:f62c53736eccb0c4934a3ea9316e0d57696bb49c1a7c86c726e9bb8a2f87dadf diff --git a/packages/pandas-gbq/.kokoro/requirements.txt b/packages/pandas-gbq/.kokoro/requirements.txt index 05dc4672edaa..096e4800a9ac 100644 --- a/packages/pandas-gbq/.kokoro/requirements.txt +++ b/packages/pandas-gbq/.kokoro/requirements.txt @@ -113,33 +113,28 @@ commonmark==0.9.1 \ --hash=sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60 \ --hash=sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9 # via rich -cryptography==38.0.3 \ - --hash=sha256:068147f32fa662c81aebab95c74679b401b12b57494872886eb5c1139250ec5d \ - --hash=sha256:06fc3cc7b6f6cca87bd56ec80a580c88f1da5306f505876a71c8cfa7050257dd \ - --hash=sha256:25c1d1f19729fb09d42e06b4bf9895212292cb27bb50229f5aa64d039ab29146 \ - --hash=sha256:402852a0aea73833d982cabb6d0c3bb582c15483d29fb7085ef2c42bfa7e38d7 \ - --hash=sha256:4e269dcd9b102c5a3d72be3c45d8ce20377b8076a43cbed6f660a1afe365e436 \ - --hash=sha256:5419a127426084933076132d317911e3c6eb77568a1ce23c3ac1e12d111e61e0 \ - --hash=sha256:554bec92ee7d1e9d10ded2f7e92a5d70c1f74ba9524947c0ba0c850c7b011828 \ - --hash=sha256:5e89468fbd2fcd733b5899333bc54d0d06c80e04cd23d8c6f3e0542358c6060b \ - --hash=sha256:65535bc550b70bd6271984d9863a37741352b4aad6fb1b3344a54e6950249b55 \ - --hash=sha256:6ab9516b85bebe7aa83f309bacc5f44a61eeb90d0b4ec125d2d003ce41932d36 \ - --hash=sha256:6addc3b6d593cd980989261dc1cce38263c76954d758c3c94de51f1e010c9a50 \ - --hash=sha256:728f2694fa743a996d7784a6194da430f197d5c58e2f4e278612b359f455e4a2 \ - --hash=sha256:785e4056b5a8b28f05a533fab69febf5004458e20dad7e2e13a3120d8ecec75a \ - --hash=sha256:78cf5eefac2b52c10398a42765bfa981ce2372cbc0457e6bf9658f41ec3c41d8 \ - --hash=sha256:7f836217000342d448e1c9a342e9163149e45d5b5eca76a30e84503a5a96cab0 \ - --hash=sha256:8d41a46251bf0634e21fac50ffd643216ccecfaf3701a063257fe0b2be1b6548 \ - --hash=sha256:984fe150f350a3c91e84de405fe49e688aa6092b3525f407a18b9646f6612320 \ - --hash=sha256:9b24bcff7853ed18a63cfb0c2b008936a9554af24af2fb146e16d8e1aed75748 \ - --hash=sha256:b1b35d9d3a65542ed2e9d90115dfd16bbc027b3f07ee3304fc83580f26e43249 \ - --hash=sha256:b1b52c9e5f8aa2b802d48bd693190341fae201ea51c7a167d69fc48b60e8a959 \ - --hash=sha256:bbf203f1a814007ce24bd4d51362991d5cb90ba0c177a9c08825f2cc304d871f \ - --hash=sha256:be243c7e2bfcf6cc4cb350c0d5cdf15ca6383bbcb2a8ef51d3c9411a9d4386f0 \ - --hash=sha256:bfbe6ee19615b07a98b1d2287d6a6073f734735b49ee45b11324d85efc4d5cbd \ - --hash=sha256:c46837ea467ed1efea562bbeb543994c2d1f6e800785bd5a2c98bc096f5cb220 \ - --hash=sha256:dfb4f4dd568de1b6af9f4cda334adf7d72cf5bc052516e1b2608b683375dd95c \ - --hash=sha256:ed7b00096790213e09eb11c97cc6e2b757f15f3d2f85833cd2d3ec3fe37c1722 +cryptography==39.0.1 \ + --hash=sha256:0f8da300b5c8af9f98111ffd512910bc792b4c77392a9523624680f7956a99d4 \ + --hash=sha256:35f7c7d015d474f4011e859e93e789c87d21f6f4880ebdc29896a60403328f1f \ + --hash=sha256:5aa67414fcdfa22cf052e640cb5ddc461924a045cacf325cd164e65312d99502 \ + --hash=sha256:5d2d8b87a490bfcd407ed9d49093793d0f75198a35e6eb1a923ce1ee86c62b41 \ + --hash=sha256:6687ef6d0a6497e2b58e7c5b852b53f62142cfa7cd1555795758934da363a965 \ + --hash=sha256:6f8ba7f0328b79f08bdacc3e4e66fb4d7aab0c3584e0bd41328dce5262e26b2e \ + --hash=sha256:706843b48f9a3f9b9911979761c91541e3d90db1ca905fd63fee540a217698bc \ + --hash=sha256:807ce09d4434881ca3a7594733669bd834f5b2c6d5c7e36f8c00f691887042ad \ + --hash=sha256:83e17b26de248c33f3acffb922748151d71827d6021d98c70e6c1a25ddd78505 \ + --hash=sha256:96f1157a7c08b5b189b16b47bc9db2332269d6680a196341bf30046330d15388 \ + --hash=sha256:aec5a6c9864be7df2240c382740fcf3b96928c46604eaa7f3091f58b878c0bb6 \ + --hash=sha256:b0afd054cd42f3d213bf82c629efb1ee5f22eba35bf0eec88ea9ea7304f511a2 \ + --hash=sha256:ced4e447ae29ca194449a3f1ce132ded8fcab06971ef5f618605aacaa612beac \ + --hash=sha256:d1f6198ee6d9148405e49887803907fe8962a23e6c6f83ea7d98f1c0de375695 \ + --hash=sha256:e124352fd3db36a9d4a21c1aa27fd5d051e621845cb87fb851c08f4f75ce8be6 \ + --hash=sha256:e422abdec8b5fa8462aa016786680720d78bdce7a30c652b7fadf83a4ba35336 \ + --hash=sha256:ef8b72fa70b348724ff1218267e7f7375b8de4e8194d1636ee60510aae104cd0 \ + --hash=sha256:f0c64d1bd842ca2633e74a1a28033d139368ad959872533b1bab8c80e8240a0c \ + --hash=sha256:f24077a3b5298a5a06a8e0536e3ea9ec60e4c7ac486755e5fb6e6ea9b3500106 \ + --hash=sha256:fdd188c8a6ef8769f148f88f859884507b954cc64db6b52f66ef199bb9ad660a \ + --hash=sha256:fe913f20024eb2cb2f323e42a64bdf2911bb9738a15dba7d3cce48151034e3a8 # via # gcp-releasetool # secretstorage From 57defb6b226465075a92a00654a0c42e3e7a7a04 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 28 Feb 2023 05:36:33 -0500 Subject: [PATCH 340/519] chore(python): upgrade gcp-releasetool in .kokoro [autoapprove] (#615) Source-Link: https://github.com/googleapis/synthtool/commit/5f2a6089f73abf06238fe4310f6a14d6f6d1eed3 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:8555f0e37e6261408f792bfd6635102d2da5ad73f8f09bcb24f25e6afb5fac97 Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 2 +- packages/pandas-gbq/.kokoro/requirements.in | 2 +- packages/pandas-gbq/.kokoro/requirements.txt | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 894fb6bc9b47..5fc5daa31783 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:f62c53736eccb0c4934a3ea9316e0d57696bb49c1a7c86c726e9bb8a2f87dadf + digest: sha256:8555f0e37e6261408f792bfd6635102d2da5ad73f8f09bcb24f25e6afb5fac97 diff --git a/packages/pandas-gbq/.kokoro/requirements.in b/packages/pandas-gbq/.kokoro/requirements.in index cbd7e77f44db..882178ce6001 100644 --- a/packages/pandas-gbq/.kokoro/requirements.in +++ b/packages/pandas-gbq/.kokoro/requirements.in @@ -1,5 +1,5 @@ gcp-docuploader -gcp-releasetool +gcp-releasetool>=1.10.5 # required for compatibility with cryptography>=39.x importlib-metadata typing-extensions twine diff --git a/packages/pandas-gbq/.kokoro/requirements.txt b/packages/pandas-gbq/.kokoro/requirements.txt index 096e4800a9ac..fa99c12908f0 100644 --- a/packages/pandas-gbq/.kokoro/requirements.txt +++ b/packages/pandas-gbq/.kokoro/requirements.txt @@ -154,9 +154,9 @@ gcp-docuploader==0.6.4 \ --hash=sha256:01486419e24633af78fd0167db74a2763974765ee8078ca6eb6964d0ebd388af \ --hash=sha256:70861190c123d907b3b067da896265ead2eeb9263969d6955c9e0bb091b5ccbf # via -r requirements.in -gcp-releasetool==1.10.0 \ - --hash=sha256:72a38ca91b59c24f7e699e9227c90cbe4dd71b789383cb0164b088abae294c83 \ - --hash=sha256:8c7c99320208383d4bb2b808c6880eb7a81424afe7cdba3c8d84b25f4f0e097d +gcp-releasetool==1.10.5 \ + --hash=sha256:174b7b102d704b254f2a26a3eda2c684fd3543320ec239baf771542a2e58e109 \ + --hash=sha256:e29d29927fe2ca493105a82958c6873bb2b90d503acac56be2c229e74de0eec9 # via -r requirements.in google-api-core==2.10.2 \ --hash=sha256:10c06f7739fe57781f87523375e8e1a3a4674bf6392cd6131a3222182b971320 \ From 72b293d236adf52d5b1d9cf0450ede2b3c563dea Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 16 Mar 2023 08:27:40 -0400 Subject: [PATCH 341/519] chore(deps): Update nox in .kokoro/requirements.in [autoapprove] (#619) Source-Link: https://github.com/googleapis/synthtool/commit/92006bb3cdc84677aa93c7f5235424ec2b157146 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:2e247c7bf5154df7f98cce087a20ca7605e236340c7d6d1a14447e5c06791bd6 Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 2 +- packages/pandas-gbq/.kokoro/requirements.in | 2 +- packages/pandas-gbq/.kokoro/requirements.txt | 14 +++++--------- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 5fc5daa31783..b8edda51cf46 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:8555f0e37e6261408f792bfd6635102d2da5ad73f8f09bcb24f25e6afb5fac97 + digest: sha256:2e247c7bf5154df7f98cce087a20ca7605e236340c7d6d1a14447e5c06791bd6 diff --git a/packages/pandas-gbq/.kokoro/requirements.in b/packages/pandas-gbq/.kokoro/requirements.in index 882178ce6001..ec867d9fd65a 100644 --- a/packages/pandas-gbq/.kokoro/requirements.in +++ b/packages/pandas-gbq/.kokoro/requirements.in @@ -5,6 +5,6 @@ typing-extensions twine wheel setuptools -nox +nox>=2022.11.21 # required to remove dependency on py charset-normalizer<3 click<8.1.0 diff --git a/packages/pandas-gbq/.kokoro/requirements.txt b/packages/pandas-gbq/.kokoro/requirements.txt index fa99c12908f0..66a2172a76a8 100644 --- a/packages/pandas-gbq/.kokoro/requirements.txt +++ b/packages/pandas-gbq/.kokoro/requirements.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.10 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: # # pip-compile --allow-unsafe --generate-hashes requirements.in # @@ -335,9 +335,9 @@ more-itertools==9.0.0 \ --hash=sha256:250e83d7e81d0c87ca6bd942e6aeab8cc9daa6096d12c5308f3f92fa5e5c1f41 \ --hash=sha256:5a6257e40878ef0520b1803990e3e22303a41b5714006c32a3fd8304b26ea1ab # via jaraco-classes -nox==2022.8.7 \ - --hash=sha256:1b894940551dc5c389f9271d197ca5d655d40bdc6ccf93ed6880e4042760a34b \ - --hash=sha256:96cca88779e08282a699d672258ec01eb7c792d35bbbf538c723172bce23212c +nox==2022.11.21 \ + --hash=sha256:0e41a990e290e274cb205a976c4c97ee3c5234441a8132c8c3fd9ea3c22149eb \ + --hash=sha256:e21c31de0711d1274ca585a2c5fde36b1aa962005ba8e9322bf5eeed16dcd684 # via -r requirements.in packaging==21.3 \ --hash=sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb \ @@ -380,10 +380,6 @@ protobuf==3.20.3 \ # gcp-docuploader # gcp-releasetool # google-api-core -py==1.11.0 \ - --hash=sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719 \ - --hash=sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378 - # via nox pyasn1==0.4.8 \ --hash=sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d \ --hash=sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba From 63a082aa7953c74a11f4f0d7f211a498670f052a Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Wed, 29 Mar 2023 12:15:54 -0400 Subject: [PATCH 342/519] docs: updates with a link to the canonical source of documentation (#620) * updates main page with a link to the canonical source of documentation * Update docs/index.rst * Update docs/index.rst Co-authored-by: Anthonios Partheniou --------- Co-authored-by: Anthonios Partheniou --- packages/pandas-gbq/docs/index.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/pandas-gbq/docs/index.rst b/packages/pandas-gbq/docs/index.rst index 5862dc17cd48..67496f2641a8 100644 --- a/packages/pandas-gbq/docs/index.rst +++ b/packages/pandas-gbq/docs/index.rst @@ -13,6 +13,10 @@ with a shape and data types derived from the source table. Additionally, DataFrames can be inserted into new BigQuery tables or appended to existing tables. +Note: The canonical version of this documentation can always be found on the +`googleapis.dev pandas-gbq site +`__. + .. note:: To use this module, you will need a valid BigQuery account. Use the From 4922fde0d66e180a03f7eb19fe4123469065b4bb Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 9 May 2023 17:24:28 -0500 Subject: [PATCH 343/519] docs: Google Colab auth is used with pydata-google-auth 1.8.0+ (#631) --- .../pandas-gbq/docs/howto/authentication.rst | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/packages/pandas-gbq/docs/howto/authentication.rst b/packages/pandas-gbq/docs/howto/authentication.rst index c65563f8d44d..17e6c04a305d 100644 --- a/packages/pandas-gbq/docs/howto/authentication.rst +++ b/packages/pandas-gbq/docs/howto/authentication.rst @@ -38,7 +38,20 @@ authentication methods: # The credentials and project_id arguments can be omitted. df = pandas_gbq.read_gbq("SELECT my_col FROM `my_dataset.my_table`") -2. Application Default Credentials via the :func:`google.auth.default` +2. If running on `Google Colab `_, + pandas-gbq attempts to authenticate with the + ``google.colab.auth.authenticate_user()`` method. See the `Getting started + with BigQuery on Colab notebook + `_ for an + example of using this authentication method with other libraries that use + Google BigQuery. + + .. note:: + + To use Colab authentication, install version 1.8.0 or later of the + ``pydata-google-auth`` package. + +3. Application Default Credentials via the :func:`google.auth.default` function. .. note:: @@ -48,10 +61,11 @@ authentication methods: user account credentials. A common problem with default credentials when running on Google - Compute Engine is that the VM does not have sufficient scopes to query - BigQuery. + Compute Engine is that the VM does not have sufficient `access scopes + `_ + to query BigQuery. -3. User account credentials. +4. User account credentials. pandas-gbq loads cached credentials from a hidden user folder on the operating system. @@ -214,5 +228,5 @@ more of the following circumstances: (or similar) notebook. If the conditions above apply to you, your needs may be better served -by the content in the `Authentication (Highly Constrained Development Environment) +by the content in the `Authentication (Highly Constrained Development Environment) `_ section. From 443f138604569be08099124cef83c6e7ab4c35d1 Mon Sep 17 00:00:00 2001 From: Charlie Schacher <18281573+quoimec@users.noreply.github.com> Date: Thu, 11 May 2023 00:09:37 +1000 Subject: [PATCH 344/519] fix: add exception context to GenericGBQExceptions (#629) Co-authored-by: Tim Swast --- packages/pandas-gbq/pandas_gbq/gbq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index c059a943c11d..edf827343120 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -393,7 +393,7 @@ def process_http_error(ex): error_message = ex.errors[0]["message"] raise TableCreationError(f"Reason: {error_message}") else: - raise GenericGBQException("Reason: {0}".format(ex)) + raise GenericGBQException("Reason: {0}".format(ex)) from ex def download_table( self, From 7b11c01d36decd1abf62da9efc93b7247ad66438 Mon Sep 17 00:00:00 2001 From: yokomotod Date: Thu, 11 May 2023 00:17:16 +0900 Subject: [PATCH 345/519] docs: correct the documented dtypes for `read_gbq` (#598) Co-authored-by: Anthonios Partheniou Co-authored-by: Tim Swast --- packages/pandas-gbq/docs/reading.rst | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/pandas-gbq/docs/reading.rst b/packages/pandas-gbq/docs/reading.rst index e3e3dc5a20bd..c5e814bfdce7 100644 --- a/packages/pandas-gbq/docs/reading.rst +++ b/packages/pandas-gbq/docs/reading.rst @@ -59,15 +59,21 @@ column, based on the BigQuery table schema. ================== ========================= BigQuery Data Type dtype ================== ========================= -DATE datetime64[ns] -DATETIME datetime64[ns] BOOL boolean -FLOAT float INT64 Int64 -TIME datetime64[ns] -TIMESTAMP :class:`~pandas.DatetimeTZDtype` with ``unit='ns'`` and ``tz='UTC'`` +FLOAT64 float64 +TIME dbtime +DATE dbdate or object +DATETIME datetime64[ns] or object +TIMESTAMP datetime64[ns, UTC] or object ================== ========================= +If any DATE/DATETIME/TIMESTAMP value is outside of the range of `pandas.Timestamp.min +`__ +(1677-09-22) and `pandas.Timestamp.max +`__ +(2262-04-11), the data type maps to the pandas `object` dtype. + .. _reading-bqstorage-api: Improving download performance From 81c49f069c924f01505a5c68c3d5574bdc0077b5 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 10 May 2023 11:01:05 -0500 Subject: [PATCH 346/519] chore(main): release 0.19.2 (#622) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- packages/pandas-gbq/CHANGELOG.md | 14 ++++++++++++++ packages/pandas-gbq/pandas_gbq/version.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index 2deaddba0885..505fa802844b 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## [0.19.2](https://github.com/googleapis/python-bigquery-pandas/compare/v0.19.1...v0.19.2) (2023-05-10) + + +### Bug Fixes + +* Add exception context to GenericGBQExceptions ([#629](https://github.com/googleapis/python-bigquery-pandas/issues/629)) ([d17ae24](https://github.com/googleapis/python-bigquery-pandas/commit/d17ae244b8e249b1d5f3b6db667db5e84abaf85c)) + + +### Documentation + +* Correct the documented dtypes for `read_gbq` ([#598](https://github.com/googleapis/python-bigquery-pandas/issues/598)) ([b45651d](https://github.com/googleapis/python-bigquery-pandas/commit/b45651d5688034418cdf57244c39592ee5dea2b2)) +* Google Colab auth is used with pydata-google-auth 1.8.0+ ([#631](https://github.com/googleapis/python-bigquery-pandas/issues/631)) ([257aa62](https://github.com/googleapis/python-bigquery-pandas/commit/257aa6209345836e291ebd4f6feec0176c8b22e1)) +* Updates with a link to the canonical source of documentation ([#620](https://github.com/googleapis/python-bigquery-pandas/issues/620)) ([1dca732](https://github.com/googleapis/python-bigquery-pandas/commit/1dca732c71e022f1897d946e2ae914e9b51bc386)) + ## [0.19.1](https://github.com/googleapis/python-bigquery-pandas/compare/v0.19.0...v0.19.1) (2023-01-25) diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py index a20f538cfecd..ee444a8007b2 100644 --- a/packages/pandas-gbq/pandas_gbq/version.py +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -2,4 +2,4 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -__version__ = "0.19.1" +__version__ = "0.19.2" From e3f7a172e7972ae274507f89f76d1bcc9a26fc9a Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 25 May 2023 13:23:08 -0400 Subject: [PATCH 347/519] build(deps): bump requests from 2.28.1 to 2.31.0 in /synthtool/gcp/templates/python_library/.kokoro (#634) Source-Link: https://github.com/googleapis/synthtool/commit/30bd01b4ab78bf1b2a425816e15b3e7e090993dd Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:9bc5fa3b62b091f60614c08a7fb4fd1d3e1678e326f34dd66ce1eefb5dc3267b Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 3 ++- packages/pandas-gbq/.kokoro/requirements.txt | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index b8edda51cf46..32b3c486591a 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,4 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:2e247c7bf5154df7f98cce087a20ca7605e236340c7d6d1a14447e5c06791bd6 + digest: sha256:9bc5fa3b62b091f60614c08a7fb4fd1d3e1678e326f34dd66ce1eefb5dc3267b +# created: 2023-05-25T14:56:16.294623272Z diff --git a/packages/pandas-gbq/.kokoro/requirements.txt b/packages/pandas-gbq/.kokoro/requirements.txt index 66a2172a76a8..3b8d7ee81848 100644 --- a/packages/pandas-gbq/.kokoro/requirements.txt +++ b/packages/pandas-gbq/.kokoro/requirements.txt @@ -419,9 +419,9 @@ readme-renderer==37.3 \ --hash=sha256:cd653186dfc73055656f090f227f5cb22a046d7f71a841dfa305f55c9a513273 \ --hash=sha256:f67a16caedfa71eef48a31b39708637a6f4664c4394801a7b0d6432d13907343 # via twine -requests==2.28.1 \ - --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983 \ - --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349 +requests==2.31.0 \ + --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \ + --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1 # via # gcp-releasetool # google-api-core From fe9dfcb290875e5a80f96b14ebc786d09c0f6022 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Sat, 3 Jun 2023 20:01:09 -0400 Subject: [PATCH 348/519] build(deps): bump cryptography from 39.0.1 to 41.0.0 in /synthtool/gcp/templates/python_library/.kokoro (#637) Source-Link: https://github.com/googleapis/synthtool/commit/d0f51a0c2a9a6bcca86911eabea9e484baadf64b Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:240b5bcc2bafd450912d2da2be15e62bc6de2cf839823ae4bf94d4f392b451dc Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 +- packages/pandas-gbq/.kokoro/requirements.txt | 42 +++++++++---------- 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 32b3c486591a..02a4dedced74 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:9bc5fa3b62b091f60614c08a7fb4fd1d3e1678e326f34dd66ce1eefb5dc3267b -# created: 2023-05-25T14:56:16.294623272Z + digest: sha256:240b5bcc2bafd450912d2da2be15e62bc6de2cf839823ae4bf94d4f392b451dc +# created: 2023-06-03T21:25:37.968717478Z diff --git a/packages/pandas-gbq/.kokoro/requirements.txt b/packages/pandas-gbq/.kokoro/requirements.txt index 3b8d7ee81848..c7929db6d152 100644 --- a/packages/pandas-gbq/.kokoro/requirements.txt +++ b/packages/pandas-gbq/.kokoro/requirements.txt @@ -113,28 +113,26 @@ commonmark==0.9.1 \ --hash=sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60 \ --hash=sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9 # via rich -cryptography==39.0.1 \ - --hash=sha256:0f8da300b5c8af9f98111ffd512910bc792b4c77392a9523624680f7956a99d4 \ - --hash=sha256:35f7c7d015d474f4011e859e93e789c87d21f6f4880ebdc29896a60403328f1f \ - --hash=sha256:5aa67414fcdfa22cf052e640cb5ddc461924a045cacf325cd164e65312d99502 \ - --hash=sha256:5d2d8b87a490bfcd407ed9d49093793d0f75198a35e6eb1a923ce1ee86c62b41 \ - --hash=sha256:6687ef6d0a6497e2b58e7c5b852b53f62142cfa7cd1555795758934da363a965 \ - --hash=sha256:6f8ba7f0328b79f08bdacc3e4e66fb4d7aab0c3584e0bd41328dce5262e26b2e \ - --hash=sha256:706843b48f9a3f9b9911979761c91541e3d90db1ca905fd63fee540a217698bc \ - --hash=sha256:807ce09d4434881ca3a7594733669bd834f5b2c6d5c7e36f8c00f691887042ad \ - --hash=sha256:83e17b26de248c33f3acffb922748151d71827d6021d98c70e6c1a25ddd78505 \ - --hash=sha256:96f1157a7c08b5b189b16b47bc9db2332269d6680a196341bf30046330d15388 \ - --hash=sha256:aec5a6c9864be7df2240c382740fcf3b96928c46604eaa7f3091f58b878c0bb6 \ - --hash=sha256:b0afd054cd42f3d213bf82c629efb1ee5f22eba35bf0eec88ea9ea7304f511a2 \ - --hash=sha256:ced4e447ae29ca194449a3f1ce132ded8fcab06971ef5f618605aacaa612beac \ - --hash=sha256:d1f6198ee6d9148405e49887803907fe8962a23e6c6f83ea7d98f1c0de375695 \ - --hash=sha256:e124352fd3db36a9d4a21c1aa27fd5d051e621845cb87fb851c08f4f75ce8be6 \ - --hash=sha256:e422abdec8b5fa8462aa016786680720d78bdce7a30c652b7fadf83a4ba35336 \ - --hash=sha256:ef8b72fa70b348724ff1218267e7f7375b8de4e8194d1636ee60510aae104cd0 \ - --hash=sha256:f0c64d1bd842ca2633e74a1a28033d139368ad959872533b1bab8c80e8240a0c \ - --hash=sha256:f24077a3b5298a5a06a8e0536e3ea9ec60e4c7ac486755e5fb6e6ea9b3500106 \ - --hash=sha256:fdd188c8a6ef8769f148f88f859884507b954cc64db6b52f66ef199bb9ad660a \ - --hash=sha256:fe913f20024eb2cb2f323e42a64bdf2911bb9738a15dba7d3cce48151034e3a8 +cryptography==41.0.0 \ + --hash=sha256:0ddaee209d1cf1f180f1efa338a68c4621154de0afaef92b89486f5f96047c55 \ + --hash=sha256:14754bcdae909d66ff24b7b5f166d69340ccc6cb15731670435efd5719294895 \ + --hash=sha256:344c6de9f8bda3c425b3a41b319522ba3208551b70c2ae00099c205f0d9fd3be \ + --hash=sha256:34d405ea69a8b34566ba3dfb0521379b210ea5d560fafedf9f800a9a94a41928 \ + --hash=sha256:3680248309d340fda9611498a5319b0193a8dbdb73586a1acf8109d06f25b92d \ + --hash=sha256:3c5ef25d060c80d6d9f7f9892e1d41bb1c79b78ce74805b8cb4aa373cb7d5ec8 \ + --hash=sha256:4ab14d567f7bbe7f1cdff1c53d5324ed4d3fc8bd17c481b395db224fb405c237 \ + --hash=sha256:5c1f7293c31ebc72163a9a0df246f890d65f66b4a40d9ec80081969ba8c78cc9 \ + --hash=sha256:6b71f64beeea341c9b4f963b48ee3b62d62d57ba93eb120e1196b31dc1025e78 \ + --hash=sha256:7d92f0248d38faa411d17f4107fc0bce0c42cae0b0ba5415505df72d751bf62d \ + --hash=sha256:8362565b3835ceacf4dc8f3b56471a2289cf51ac80946f9087e66dc283a810e0 \ + --hash=sha256:84a165379cb9d411d58ed739e4af3396e544eac190805a54ba2e0322feb55c46 \ + --hash=sha256:88ff107f211ea696455ea8d911389f6d2b276aabf3231bf72c8853d22db755c5 \ + --hash=sha256:9f65e842cb02550fac96536edb1d17f24c0a338fd84eaf582be25926e993dde4 \ + --hash=sha256:a4fc68d1c5b951cfb72dfd54702afdbbf0fb7acdc9b7dc4301bbf2225a27714d \ + --hash=sha256:b7f2f5c525a642cecad24ee8670443ba27ac1fab81bba4cc24c7b6b41f2d0c75 \ + --hash=sha256:b846d59a8d5a9ba87e2c3d757ca019fa576793e8758174d3868aecb88d6fc8eb \ + --hash=sha256:bf8fc66012ca857d62f6a347007e166ed59c0bc150cefa49f28376ebe7d992a2 \ + --hash=sha256:f5d0bf9b252f30a31664b6f64432b4730bb7038339bd18b1fafe129cfc2be9be # via # gcp-releasetool # secretstorage From aafa9c994cad5ec1a42aec37e6a7677295b5f519 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 27 Jun 2023 10:11:13 -0400 Subject: [PATCH 349/519] chore: remove pinned Sphinx version [autoapprove] (#640) Source-Link: https://github.com/googleapis/synthtool/commit/909573ce9da2819eeb835909c795d29aea5c724e Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:ddf4551385d566771dc713090feb7b4c1164fb8a698fe52bbe7670b24236565b Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 ++-- packages/pandas-gbq/noxfile.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 02a4dedced74..1b3cb6c52663 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:240b5bcc2bafd450912d2da2be15e62bc6de2cf839823ae4bf94d4f392b451dc -# created: 2023-06-03T21:25:37.968717478Z + digest: sha256:ddf4551385d566771dc713090feb7b4c1164fb8a698fe52bbe7670b24236565b +# created: 2023-06-27T13:04:21.96690344Z diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index e18072ca0ec1..2b876b2e1354 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -396,10 +396,9 @@ def docfx(session): session.install("-e", ".") session.install( - "sphinx==4.0.1", + "gcp-sphinx-docfx-yaml", "alabaster", "recommonmark", - "gcp-sphinx-docfx-yaml", ) shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) From c3f1aa6aa82dafda9af54015c89524cdcac4b841 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 29 Jun 2023 12:33:55 -0400 Subject: [PATCH 350/519] chore: store artifacts in placer (#641) Source-Link: https://github.com/googleapis/synthtool/commit/cb960373d12d20f8dc38beee2bf884d49627165e Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:2d816f26f728ac8b24248741e7d4c461c09764ef9f7be3684d557c9632e46dbd Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 ++-- packages/pandas-gbq/.kokoro/release/common.cfg | 9 +++++++++ packages/pandas-gbq/noxfile.py | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 1b3cb6c52663..98994f474104 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:ddf4551385d566771dc713090feb7b4c1164fb8a698fe52bbe7670b24236565b -# created: 2023-06-27T13:04:21.96690344Z + digest: sha256:2d816f26f728ac8b24248741e7d4c461c09764ef9f7be3684d557c9632e46dbd +# created: 2023-06-28T17:03:33.371210701Z diff --git a/packages/pandas-gbq/.kokoro/release/common.cfg b/packages/pandas-gbq/.kokoro/release/common.cfg index 0fd554538846..7b60d2107274 100644 --- a/packages/pandas-gbq/.kokoro/release/common.cfg +++ b/packages/pandas-gbq/.kokoro/release/common.cfg @@ -38,3 +38,12 @@ env_vars: { key: "SECRET_MANAGER_KEYS" value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem" } + +# Store the packages we uploaded to PyPI. That way, we have a record of exactly +# what we published, which we can use to generate SBOMs and attestations. +action { + define_artifacts { + regex: "github/python-bigquery-pandas/**/*.tar.gz" + strip_prefix: "github/python-bigquery-pandas" + } +} diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 2b876b2e1354..7ccecaac90b4 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -470,6 +470,7 @@ def prerelease_deps(session): "grpcio!=1.52.0rc1", "grpcio-status", "google-api-core", + "google-auth", "proto-plus", "google-cloud-testutils", # dependencies of google-cloud-testutils" @@ -482,7 +483,6 @@ def prerelease_deps(session): # Remaining dependencies other_deps = [ "requests", - "google-auth", ] session.install(*other_deps) From 0542d110f8c468c795e1664397424762961c6f35 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Thu, 6 Jul 2023 01:24:34 +0200 Subject: [PATCH 351/519] chore(deps): update all dependencies (#571) --- .../pandas-gbq/samples/snippets/requirements-test.txt | 2 +- packages/pandas-gbq/samples/snippets/requirements.txt | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements-test.txt b/packages/pandas-gbq/samples/snippets/requirements-test.txt index 8394ed528160..2a4dccc0df2b 100644 --- a/packages/pandas-gbq/samples/snippets/requirements-test.txt +++ b/packages/pandas-gbq/samples/snippets/requirements-test.txt @@ -1,2 +1,2 @@ google-cloud-testutils==1.3.3 -pytest==7.1.3 +pytest==7.4.0 diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 6a681437d6c9..b0b76cd84c28 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,6 +1,6 @@ -google-cloud-bigquery-storage==2.15.0 -google-cloud-bigquery==3.3.2 -pandas-gbq==0.17.8 +google-cloud-bigquery-storage==2.21.0 +google-cloud-bigquery==3.11.3 +pandas-gbq==0.19.2 pandas===1.3.5; python_version == '3.7' -pandas==1.4.4; python_version >= '3.8' -pyarrow==9.0.0 +pandas==2.0.3; python_version >= '3.8' +pyarrow==12.0.1 From 792d4bf4ff208ac95716e8010ed1e11ed8991af7 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Thu, 6 Jul 2023 20:48:07 +0200 Subject: [PATCH 352/519] chore(deps): update dependency google-cloud-bigquery-storage to v2.22.0 (#642) --- packages/pandas-gbq/samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index b0b76cd84c28..f01ccb1b7f20 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,4 +1,4 @@ -google-cloud-bigquery-storage==2.21.0 +google-cloud-bigquery-storage==2.22.0 google-cloud-bigquery==3.11.3 pandas-gbq==0.19.2 pandas===1.3.5; python_version == '3.7' From d6e3c0d75ee3d13c502618a66b5484249a990b8c Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Fri, 14 Jul 2023 11:25:26 -0400 Subject: [PATCH 353/519] feat: migrating off of circle/ci (#638) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * saving state of noxfile prior to owlbot.py edits * minor clean up of leftover artifacts from various experiments * linting and black * adds noxfile.py to excludes and removes noxfile edits in owlbot * Adds new kokoro file in presubmit * removes circle/ci config file * Update noxfile.py * Update noxfile.py * Update noxfile.py * remove limit on printing only KOKORO env variables (for testing) * updates build.sh with conda environmental variables * updates build.sh with conda environmental variables redux * removes the ci folder and files related to circle ci * updates env variable to display all vars not just KOKORO * removes additional content related to circle ci * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * updates owlbot with build.sh edits * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * updates owlbot with build.sh minor tweak * revert build.sh * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * testing regex change * revert build.sh * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot --- packages/pandas-gbq/.circleci/config.yml | 34 ----- packages/pandas-gbq/.kokoro/build.sh | 11 +- .../.kokoro/presubmit/conda_test.cfg | 7 + packages/pandas-gbq/ci/config_auth.sh | 13 -- .../ci/requirements-3.7-0.24.2.conda | 17 --- .../ci/requirements-3.9-1.3.4.conda | 14 -- packages/pandas-gbq/ci/run_conda.sh | 23 --- packages/pandas-gbq/ci/run_tests.sh | 15 -- packages/pandas-gbq/noxfile.py | 81 +++++++++++ packages/pandas-gbq/owlbot.py | 133 ++++-------------- packages/pandas-gbq/tests/system/conftest.py | 11 -- packages/pandas-gbq/tests/system/test_auth.py | 2 +- 12 files changed, 123 insertions(+), 238 deletions(-) delete mode 100644 packages/pandas-gbq/.circleci/config.yml create mode 100644 packages/pandas-gbq/.kokoro/presubmit/conda_test.cfg delete mode 100755 packages/pandas-gbq/ci/config_auth.sh delete mode 100644 packages/pandas-gbq/ci/requirements-3.7-0.24.2.conda delete mode 100644 packages/pandas-gbq/ci/requirements-3.9-1.3.4.conda delete mode 100755 packages/pandas-gbq/ci/run_conda.sh delete mode 100755 packages/pandas-gbq/ci/run_tests.sh diff --git a/packages/pandas-gbq/.circleci/config.yml b/packages/pandas-gbq/.circleci/config.yml deleted file mode 100644 index e008054c67ad..000000000000 --- a/packages/pandas-gbq/.circleci/config.yml +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) 2017 pandas-gbq Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -version: 2 -jobs: - # Conda - "conda-3.7": - docker: - - image: mambaorg/micromamba - environment: - PYTHON: "3.7" - PANDAS: "0.24.2" - steps: - - checkout - - run: ci/config_auth.sh - - run: ci/run_conda.sh - "conda-3.9": - docker: - - image: mambaorg/micromamba - environment: - PYTHON: "3.9" - PANDAS: "1.3.4" - steps: - - checkout - - run: ci/config_auth.sh - - run: ci/run_conda.sh - -workflows: - version: 2 - build: - jobs: - - "conda-3.7" - - "conda-3.9" diff --git a/packages/pandas-gbq/.kokoro/build.sh b/packages/pandas-gbq/.kokoro/build.sh index d02a78db4dab..204ea03cf6bd 100755 --- a/packages/pandas-gbq/.kokoro/build.sh +++ b/packages/pandas-gbq/.kokoro/build.sh @@ -23,9 +23,18 @@ cd "${PROJECT_ROOT}" # Disable buffering, so that the logs stream through. export PYTHONUNBUFFERED=1 +export CONDA_EXE=/root/conda/bin/conda +export CONDA_PREFIX=/root/conda +export CONDA_PROMPT_MODIFIER=(base) +export _CE_CONDA= +export CONDA_SHLVL=1 +export CONDA_PYTHON_EXE=/root/conda/bin/python +export CONDA_DEFAULT_ENV=base +export PATH=/root/conda/bin:/root/conda/condabin:${PATH} + # Debug: show build environment -env | grep KOKORO +env # Setup service account credentials. export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json diff --git a/packages/pandas-gbq/.kokoro/presubmit/conda_test.cfg b/packages/pandas-gbq/.kokoro/presubmit/conda_test.cfg new file mode 100644 index 000000000000..6e3943f35021 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/presubmit/conda_test.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "conda_test" +} diff --git a/packages/pandas-gbq/ci/config_auth.sh b/packages/pandas-gbq/ci/config_auth.sh deleted file mode 100755 index cde115c72d29..000000000000 --- a/packages/pandas-gbq/ci/config_auth.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -# Copyright (c) 2017 pandas-gbq Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -set -e -# Don't set -x, because we don't want to leak keys. -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -# Write key to file if present. -if [ ! -z "$SERVICE_ACCOUNT_KEY" ] ; then - echo "$SERVICE_ACCOUNT_KEY" | base64 --decode > "$DIR"/service_account.json -fi diff --git a/packages/pandas-gbq/ci/requirements-3.7-0.24.2.conda b/packages/pandas-gbq/ci/requirements-3.7-0.24.2.conda deleted file mode 100644 index 2d61383ea2a1..000000000000 --- a/packages/pandas-gbq/ci/requirements-3.7-0.24.2.conda +++ /dev/null @@ -1,17 +0,0 @@ -codecov -coverage -db-dtypes -fastavro -flake8 -freezegun -numpy -google-api-core -google-auth -google-cloud-bigquery -google-cloud-bigquery-storage -pyarrow -pydata-google-auth -pytest -pytest-cov -requests-oauthlib -tqdm diff --git a/packages/pandas-gbq/ci/requirements-3.9-1.3.4.conda b/packages/pandas-gbq/ci/requirements-3.9-1.3.4.conda deleted file mode 100644 index 1411fe5b06d7..000000000000 --- a/packages/pandas-gbq/ci/requirements-3.9-1.3.4.conda +++ /dev/null @@ -1,14 +0,0 @@ -codecov -coverage -db-dtypes -fastavro -flake8 -freezegun -google-cloud-bigquery -google-cloud-bigquery-storage -numpy -pyarrow -pydata-google-auth -pytest -pytest-cov -tqdm diff --git a/packages/pandas-gbq/ci/run_conda.sh b/packages/pandas-gbq/ci/run_conda.sh deleted file mode 100755 index 11b5b569fff2..000000000000 --- a/packages/pandas-gbq/ci/run_conda.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# Copyright (c) 2017 pandas-gbq Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -set -e -x -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -eval "$(micromamba shell hook --shell=bash)" -micromamba activate - -# Install dependencies using (micro)mamba -# https://github.com/mamba-org/micromamba-docker -REQ="ci/requirements-${PYTHON}-${PANDAS}" -micromamba install -q pandas=$PANDAS python=${PYTHON} -c conda-forge; -micromamba install -q --file "$REQ.conda" -c conda-forge; -micromamba list -micromamba info - -python setup.py develop --no-deps - -# Run the tests -$DIR/run_tests.sh diff --git a/packages/pandas-gbq/ci/run_tests.sh b/packages/pandas-gbq/ci/run_tests.sh deleted file mode 100755 index 8a1d7f911355..000000000000 --- a/packages/pandas-gbq/ci/run_tests.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# Copyright (c) 2017 pandas-gbq Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -set -e -x -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -if [ -f "$DIR/service_account.json" ]; then - export GOOGLE_APPLICATION_CREDENTIALS="$DIR/service_account.json" -fi - -# Install test requirements -pip install coverage pytest pytest-cov flake8 codecov google-cloud-testutils -pytest -v -m "not local_auth" --cov=pandas_gbq --cov-report xml:/tmp/pytest-cov.xml --cov-fail-under=0 tests diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 7ccecaac90b4..135ce432c5cd 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -31,6 +31,7 @@ DEFAULT_PYTHON_VERSION = "3.8" + UNIT_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11"] UNIT_TEST_STANDARD_DEPENDENCIES = [ "mock", @@ -51,6 +52,11 @@ "3.9": [], } +CONDA_TEST_PYTHON_VERSIONS = [ + UNIT_TEST_PYTHON_VERSIONS[0], + UNIT_TEST_PYTHON_VERSIONS[-1], +] + SYSTEM_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11"] SYSTEM_TEST_STANDARD_DEPENDENCIES = [ "mock", @@ -514,3 +520,78 @@ def prerelease_deps(session): system_test_folder_path, *session.posargs, ) + + +def install_conda_unittest_dependencies(session, standard_deps, conda_forge_packages): + """Installs packages from conda forge, pypi, and locally.""" + + # Install from conda-forge and default conda package repos. + session.conda_install(*conda_forge_packages, channel=["defaults", "conda-forge"]) + + # Install from pypi for packages not readily available on conda forge. + session.install( + *standard_deps, + ) + + # Install via pip from the local repo, avoid doing dependency resolution + # via pip, so that we don't override any conda resolved dependencies + session.install("-e", ".", "--no-deps") + + +@nox.session(python=CONDA_TEST_PYTHON_VERSIONS, venv_backend="mamba") +def conda_test(session): + """Run test suite in a conda virtual environment. + + Installs all test dependencies, then installs this package. + NOTE: Some of these libraries are not readily available on conda-forge + at this time and are thus installed using pip after the base install of + libraries from conda-forge. + + We decided that it was more important to prove a base ability to install + using conda than to complicate things with adding a whole nother + set of constraints just for a conda install, so this install does not + attempt to constrain packages (i.e. in a constraints-x.x.txt file) + manually. + """ + + standard_deps = ( + UNIT_TEST_STANDARD_DEPENDENCIES + + UNIT_TEST_DEPENDENCIES + + UNIT_TEST_EXTERNAL_DEPENDENCIES + ) + + conda_forge_packages = [ + "db-dtypes", + "google-api-core", + "google-auth", + "google-auth-oauthlib", + "google-cloud-bigquery", + "google-cloud-bigquery-storage", + "numpy", + "pandas", + "pyarrow", + "pydata-google-auth", + "tqdm", + "protobuf", + ] + + install_conda_unittest_dependencies(session, standard_deps, conda_forge_packages) + + # Provide a list of all installed packages (both from conda forge and pip) + # for troubleshooting purposes. + session.run("mamba", "list") + + # Tests are limited to unit tests only, at this time. + session.run( + "py.test", + "--quiet", + f"--junitxml=unit_{session.python}_sponge_log.xml", + "--cov=pandas_gbq", + "--cov=tests/unit", + "--cov-append", + "--cov-config=.coveragerc", + "--cov-report=", + "--cov-fail-under=0", + os.path.join("tests", "unit"), + *session.posargs, + ) diff --git a/packages/pandas-gbq/owlbot.py b/packages/pandas-gbq/owlbot.py index 9c9454f8852f..e648334d7996 100644 --- a/packages/pandas-gbq/owlbot.py +++ b/packages/pandas-gbq/owlbot.py @@ -52,10 +52,11 @@ excludes=[ # pandas-gbq was originally licensed BSD-3-Clause License "LICENSE", - # Mulit-processing note isn't relevant, as pandas_gbq is responsible for + # Multi-processing note isn't relevant, as pandas_gbq is responsible for # creating clients, not the end user. "docs/multiprocessing.rst", - "README.rst", + "noxfile.py", + "README.rst", ], ) @@ -64,123 +65,37 @@ # ---------------------------------------------------------------------------- s.replace( - ["noxfile.py"], - r"import pathlib\s+import shutil", - "import pathlib\nimport re\nimport shutil", -) - -s.replace( - ["noxfile.py"], r"[\"']google[\"']", '"pandas_gbq"', + [".github/header-checker-lint.yml"], '"Google LLC"', '"pandas-gbq Authors"', ) +# Work around bug in templates https://github.com/googleapis/synthtool/pull/1335 +s.replace(".github/workflows/unittest.yml", "--fail-under=100", "--fail-under=96") +# Add environment variables to build.sh to support conda virtualenv +# installations s.replace( - ["noxfile.py"], "--cov=google", "--cov=pandas_gbq", -) - -# Workaround for https://github.com/googleapis/synthtool/issues/1317 -s.replace( - ["noxfile.py"], r'extras = "\[\]"', 'extras = ""', + [".kokoro/build.sh"], + "export PYTHONUNBUFFERED=1", + r"""export PYTHONUNBUFFERED=1 +export CONDA_EXE=/root/conda/bin/conda +export CONDA_PREFIX=/root/conda +export CONDA_PROMPT_MODIFIER=(base) +export _CE_CONDA= +export CONDA_SHLVL=1 +export CONDA_PYTHON_EXE=/root/conda/bin/python +export CONDA_DEFAULT_ENV=base +export PATH=/root/conda/bin:/root/conda/condabin:${PATH} +""", ) -s.replace( - ["noxfile.py"], - r"@nox.session\(python=DEFAULT_PYTHON_VERSION\)\s+def cover\(session\):", - r"""@nox.session(python=DEFAULT_PYTHON_VERSION) -def prerelease(session): - session.install( - "--extra-index-url", - "https://pypi.fury.io/arrow-nightlies/", - "--prefer-binary", - "--pre", - "--upgrade", - "pyarrow", - ) - session.install( - "--extra-index-url", - "https://pypi.anaconda.org/scipy-wheels-nightly/simple", - "--prefer-binary", - "--pre", - "--upgrade", - "pandas", - ) - session.install( - "--prefer-binary", - "--pre", - "--upgrade", - "google-api-core", - "google-cloud-bigquery", - "google-cloud-bigquery-storage", - "google-cloud-core", - "google-resumable-media", - # Exclude version 1.49.0rc1 which has a known issue. See https://github.com/grpc/grpc/pull/30642 - "grpcio!=1.49.0rc1", - ) - session.install( - "freezegun", - "google-cloud-datacatalog", - "google-cloud-storage", - "google-cloud-testutils", - "IPython", - "mock", - "psutil", - "pytest", - "pytest-cov", - ) - - # Because we test minimum dependency versions on the minimum Python - # version, the first version we test with in the unit tests sessions has a - # constraints file containing all dependencies and extras. - with open( - CURRENT_DIRECTORY - / "testing" - / f"constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt", - encoding="utf-8", - ) as constraints_file: - constraints_text = constraints_file.read() - - # Ignore leading whitespace and comment lines. - deps = [ - match.group(1) - for match in re.finditer( - r"^\\s*(\\S+)(?===\\S+)", constraints_text, flags=re.MULTILINE - ) - ] - - # We use --no-deps to ensure that pre-release versions aren't overwritten - # by the version ranges in setup.py. - session.install(*deps) - session.install("--no-deps", "-e", ".[all]") - - # Print out prerelease package versions. - session.run("python", "-m", "pip", "freeze") - - # Run all tests, except a few samples tests which require extra dependencies. - session.run( - "py.test", - "--quiet", - f"--junitxml=prerelease_unit_{session.python}_sponge_log.xml", - os.path.join("tests", "unit"), - ) - session.run( - "py.test", - "--quiet", - f"--junitxml=prerelease_system_{session.python}_sponge_log.xml", - os.path.join("tests", "system"), - ) - - -@nox.session(python=DEFAULT_PYTHON_VERSION) -def cover(session):""", - re.MULTILINE, -) +# Enable display of all environmental variables, not just KOKORO related vars s.replace( - [".github/header-checker-lint.yml"], '"Google LLC"', '"pandas-gbq Authors"', + [".kokoro/build.sh"], + r"env \| grep KOKORO", + "env", ) -# Work around bug in templates https://github.com/googleapis/synthtool/pull/1335 -s.replace(".github/workflows/unittest.yml", "--fail-under=100", "--fail-under=96") # ---------------------------------------------------------------------------- # Samples templates diff --git a/packages/pandas-gbq/tests/system/conftest.py b/packages/pandas-gbq/tests/system/conftest.py index 4ba8bf31008f..9690446dc762 100644 --- a/packages/pandas-gbq/tests/system/conftest.py +++ b/packages/pandas-gbq/tests/system/conftest.py @@ -16,17 +16,6 @@ REPO_DIR = pathlib.Path(__file__).parent.parent.parent -# TODO: remove when fully migrated off of Circle CI -@pytest.fixture(scope="session", autouse=True) -def default_credentials(): - """Setup application default credentials for use in code samples.""" - # Written by the 'ci/config_auth.sh' script. - path = REPO_DIR / "ci" / "service_account.json" - - if path.is_file() and "GOOGLE_APPLICATION_CREDENTIALS" not in os.environ: - os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = str(path) - - @pytest.fixture(scope="session", autouse=True) def cleanup_datasets(bigquery_client: bigquery.Client): for dataset in bigquery_client.list_datasets(): diff --git a/packages/pandas-gbq/tests/system/test_auth.py b/packages/pandas-gbq/tests/system/test_auth.py index d9f7d0965469..ecedd9732256 100644 --- a/packages/pandas-gbq/tests/system/test_auth.py +++ b/packages/pandas-gbq/tests/system/test_auth.py @@ -12,7 +12,7 @@ from pandas_gbq import auth -IS_RUNNING_ON_CI = "CIRCLE_BUILD_NUM" in os.environ or "KOKORO_BUILD_ID" in os.environ +IS_RUNNING_ON_CI = "KOKORO_BUILD_ID" in os.environ def mock_default_credentials(scopes=None, request=None): From 446f47913ce0546baaa4c4eb554cef0ce370889a Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Mon, 17 Jul 2023 13:04:42 -0400 Subject: [PATCH 354/519] build(deps): [autoapprove] bump cryptography from 41.0.0 to 41.0.2 (#652) Source-Link: https://github.com/googleapis/synthtool/commit/d6103f4a3540ba60f633a9e25c37ec5fe7e6286d Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:39f0f3f2be02ef036e297e376fe3b6256775576da8a6ccb1d5eeb80f4c8bf8fb Co-authored-by: Owl Bot --- packages/pandas-gbq/.coveragerc | 2 +- packages/pandas-gbq/.flake8 | 2 +- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 +- packages/pandas-gbq/.github/auto-label.yaml | 2 +- packages/pandas-gbq/.kokoro/build.sh | 2 +- .../pandas-gbq/.kokoro/docker/docs/Dockerfile | 2 +- .../pandas-gbq/.kokoro/populate-secrets.sh | 2 +- packages/pandas-gbq/.kokoro/publish-docs.sh | 2 +- packages/pandas-gbq/.kokoro/release.sh | 2 +- packages/pandas-gbq/.kokoro/requirements.txt | 44 ++++++++++--------- .../.kokoro/test-samples-against-head.sh | 2 +- .../pandas-gbq/.kokoro/test-samples-impl.sh | 2 +- packages/pandas-gbq/.kokoro/test-samples.sh | 2 +- packages/pandas-gbq/.kokoro/trampoline.sh | 2 +- packages/pandas-gbq/.kokoro/trampoline_v2.sh | 2 +- packages/pandas-gbq/.pre-commit-config.yaml | 2 +- packages/pandas-gbq/.trampolinerc | 4 +- packages/pandas-gbq/MANIFEST.in | 2 +- packages/pandas-gbq/docs/conf.py | 2 +- .../pandas-gbq/scripts/decrypt-secrets.sh | 2 +- .../scripts/readme-gen/readme_gen.py | 18 ++++---- packages/pandas-gbq/setup.cfg | 2 +- 22 files changed, 54 insertions(+), 52 deletions(-) diff --git a/packages/pandas-gbq/.coveragerc b/packages/pandas-gbq/.coveragerc index d6261761e1f4..6948e4d98345 100644 --- a/packages/pandas-gbq/.coveragerc +++ b/packages/pandas-gbq/.coveragerc @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright 2020 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/.flake8 b/packages/pandas-gbq/.flake8 index 2e438749863d..87f6e408c47d 100644 --- a/packages/pandas-gbq/.flake8 +++ b/packages/pandas-gbq/.flake8 @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright 2020 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 98994f474104..ae4a522b9e5f 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:2d816f26f728ac8b24248741e7d4c461c09764ef9f7be3684d557c9632e46dbd -# created: 2023-06-28T17:03:33.371210701Z + digest: sha256:39f0f3f2be02ef036e297e376fe3b6256775576da8a6ccb1d5eeb80f4c8bf8fb +# created: 2023-07-17T15:20:13.819193964Z diff --git a/packages/pandas-gbq/.github/auto-label.yaml b/packages/pandas-gbq/.github/auto-label.yaml index 41bff0b5375a..b2016d119b40 100644 --- a/packages/pandas-gbq/.github/auto-label.yaml +++ b/packages/pandas-gbq/.github/auto-label.yaml @@ -1,4 +1,4 @@ -# Copyright 2022 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/.kokoro/build.sh b/packages/pandas-gbq/.kokoro/build.sh index 204ea03cf6bd..9abf1e992257 100755 --- a/packages/pandas-gbq/.kokoro/build.sh +++ b/packages/pandas-gbq/.kokoro/build.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2018 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile b/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile index f8137d0ae497..8e39a2cc438d 100644 --- a/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile +++ b/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2020 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/.kokoro/populate-secrets.sh b/packages/pandas-gbq/.kokoro/populate-secrets.sh index f52514257ef0..6f3972140e80 100755 --- a/packages/pandas-gbq/.kokoro/populate-secrets.sh +++ b/packages/pandas-gbq/.kokoro/populate-secrets.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2020 Google LLC. +# Copyright 2023 Google LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/.kokoro/publish-docs.sh b/packages/pandas-gbq/.kokoro/publish-docs.sh index 1c4d62370042..9eafe0be3bba 100755 --- a/packages/pandas-gbq/.kokoro/publish-docs.sh +++ b/packages/pandas-gbq/.kokoro/publish-docs.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2020 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/.kokoro/release.sh b/packages/pandas-gbq/.kokoro/release.sh index 56d7f68565e1..7502861622d5 100755 --- a/packages/pandas-gbq/.kokoro/release.sh +++ b/packages/pandas-gbq/.kokoro/release.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2020 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/.kokoro/requirements.txt b/packages/pandas-gbq/.kokoro/requirements.txt index c7929db6d152..67d70a110897 100644 --- a/packages/pandas-gbq/.kokoro/requirements.txt +++ b/packages/pandas-gbq/.kokoro/requirements.txt @@ -113,26 +113,30 @@ commonmark==0.9.1 \ --hash=sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60 \ --hash=sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9 # via rich -cryptography==41.0.0 \ - --hash=sha256:0ddaee209d1cf1f180f1efa338a68c4621154de0afaef92b89486f5f96047c55 \ - --hash=sha256:14754bcdae909d66ff24b7b5f166d69340ccc6cb15731670435efd5719294895 \ - --hash=sha256:344c6de9f8bda3c425b3a41b319522ba3208551b70c2ae00099c205f0d9fd3be \ - --hash=sha256:34d405ea69a8b34566ba3dfb0521379b210ea5d560fafedf9f800a9a94a41928 \ - --hash=sha256:3680248309d340fda9611498a5319b0193a8dbdb73586a1acf8109d06f25b92d \ - --hash=sha256:3c5ef25d060c80d6d9f7f9892e1d41bb1c79b78ce74805b8cb4aa373cb7d5ec8 \ - --hash=sha256:4ab14d567f7bbe7f1cdff1c53d5324ed4d3fc8bd17c481b395db224fb405c237 \ - --hash=sha256:5c1f7293c31ebc72163a9a0df246f890d65f66b4a40d9ec80081969ba8c78cc9 \ - --hash=sha256:6b71f64beeea341c9b4f963b48ee3b62d62d57ba93eb120e1196b31dc1025e78 \ - --hash=sha256:7d92f0248d38faa411d17f4107fc0bce0c42cae0b0ba5415505df72d751bf62d \ - --hash=sha256:8362565b3835ceacf4dc8f3b56471a2289cf51ac80946f9087e66dc283a810e0 \ - --hash=sha256:84a165379cb9d411d58ed739e4af3396e544eac190805a54ba2e0322feb55c46 \ - --hash=sha256:88ff107f211ea696455ea8d911389f6d2b276aabf3231bf72c8853d22db755c5 \ - --hash=sha256:9f65e842cb02550fac96536edb1d17f24c0a338fd84eaf582be25926e993dde4 \ - --hash=sha256:a4fc68d1c5b951cfb72dfd54702afdbbf0fb7acdc9b7dc4301bbf2225a27714d \ - --hash=sha256:b7f2f5c525a642cecad24ee8670443ba27ac1fab81bba4cc24c7b6b41f2d0c75 \ - --hash=sha256:b846d59a8d5a9ba87e2c3d757ca019fa576793e8758174d3868aecb88d6fc8eb \ - --hash=sha256:bf8fc66012ca857d62f6a347007e166ed59c0bc150cefa49f28376ebe7d992a2 \ - --hash=sha256:f5d0bf9b252f30a31664b6f64432b4730bb7038339bd18b1fafe129cfc2be9be +cryptography==41.0.2 \ + --hash=sha256:01f1d9e537f9a15b037d5d9ee442b8c22e3ae11ce65ea1f3316a41c78756b711 \ + --hash=sha256:079347de771f9282fbfe0e0236c716686950c19dee1b76240ab09ce1624d76d7 \ + --hash=sha256:182be4171f9332b6741ee818ec27daff9fb00349f706629f5cbf417bd50e66fd \ + --hash=sha256:192255f539d7a89f2102d07d7375b1e0a81f7478925b3bc2e0549ebf739dae0e \ + --hash=sha256:2a034bf7d9ca894720f2ec1d8b7b5832d7e363571828037f9e0c4f18c1b58a58 \ + --hash=sha256:342f3767e25876751e14f8459ad85e77e660537ca0a066e10e75df9c9e9099f0 \ + --hash=sha256:439c3cc4c0d42fa999b83ded80a9a1fb54d53c58d6e59234cfe97f241e6c781d \ + --hash=sha256:49c3222bb8f8e800aead2e376cbef687bc9e3cb9b58b29a261210456a7783d83 \ + --hash=sha256:674b669d5daa64206c38e507808aae49904c988fa0a71c935e7006a3e1e83831 \ + --hash=sha256:7a9a3bced53b7f09da251685224d6a260c3cb291768f54954e28f03ef14e3766 \ + --hash=sha256:7af244b012711a26196450d34f483357e42aeddb04128885d95a69bd8b14b69b \ + --hash=sha256:7d230bf856164de164ecb615ccc14c7fc6de6906ddd5b491f3af90d3514c925c \ + --hash=sha256:84609ade00a6ec59a89729e87a503c6e36af98ddcd566d5f3be52e29ba993182 \ + --hash=sha256:9a6673c1828db6270b76b22cc696f40cde9043eb90373da5c2f8f2158957f42f \ + --hash=sha256:9b6d717393dbae53d4e52684ef4f022444fc1cce3c48c38cb74fca29e1f08eaa \ + --hash=sha256:9c3fe6534d59d071ee82081ca3d71eed3210f76ebd0361798c74abc2bcf347d4 \ + --hash=sha256:a719399b99377b218dac6cf547b6ec54e6ef20207b6165126a280b0ce97e0d2a \ + --hash=sha256:b332cba64d99a70c1e0836902720887fb4529ea49ea7f5462cf6640e095e11d2 \ + --hash=sha256:d124682c7a23c9764e54ca9ab5b308b14b18eba02722b8659fb238546de83a76 \ + --hash=sha256:d73f419a56d74fef257955f51b18d046f3506270a5fd2ac5febbfa259d6c0fa5 \ + --hash=sha256:f0dc40e6f7aa37af01aba07277d3d64d5a03dc66d682097541ec4da03cc140ee \ + --hash=sha256:f14ad275364c8b4e525d018f6716537ae7b6d369c094805cae45300847e0894f \ + --hash=sha256:f772610fe364372de33d76edcd313636a25684edb94cee53fd790195f5989d14 # via # gcp-releasetool # secretstorage diff --git a/packages/pandas-gbq/.kokoro/test-samples-against-head.sh b/packages/pandas-gbq/.kokoro/test-samples-against-head.sh index ba3a707b040c..63ac41dfae1d 100755 --- a/packages/pandas-gbq/.kokoro/test-samples-against-head.sh +++ b/packages/pandas-gbq/.kokoro/test-samples-against-head.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2020 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/.kokoro/test-samples-impl.sh b/packages/pandas-gbq/.kokoro/test-samples-impl.sh index 2c6500cae0b9..5a0f5fab6a89 100755 --- a/packages/pandas-gbq/.kokoro/test-samples-impl.sh +++ b/packages/pandas-gbq/.kokoro/test-samples-impl.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2021 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/.kokoro/test-samples.sh b/packages/pandas-gbq/.kokoro/test-samples.sh index 11c042d342d7..50b35a48c190 100755 --- a/packages/pandas-gbq/.kokoro/test-samples.sh +++ b/packages/pandas-gbq/.kokoro/test-samples.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2020 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/.kokoro/trampoline.sh b/packages/pandas-gbq/.kokoro/trampoline.sh index f39236e943a8..d85b1f267693 100755 --- a/packages/pandas-gbq/.kokoro/trampoline.sh +++ b/packages/pandas-gbq/.kokoro/trampoline.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2017 Google Inc. +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/.kokoro/trampoline_v2.sh b/packages/pandas-gbq/.kokoro/trampoline_v2.sh index 4af6cdc26dbc..59a7cf3a9373 100755 --- a/packages/pandas-gbq/.kokoro/trampoline_v2.sh +++ b/packages/pandas-gbq/.kokoro/trampoline_v2.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright 2020 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/.pre-commit-config.yaml b/packages/pandas-gbq/.pre-commit-config.yaml index 5405cc8ff1f3..9e3898fd1c12 100644 --- a/packages/pandas-gbq/.pre-commit-config.yaml +++ b/packages/pandas-gbq/.pre-commit-config.yaml @@ -1,4 +1,4 @@ -# Copyright 2021 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/.trampolinerc b/packages/pandas-gbq/.trampolinerc index 0eee72ab62aa..a7dfeb42c6d0 100644 --- a/packages/pandas-gbq/.trampolinerc +++ b/packages/pandas-gbq/.trampolinerc @@ -1,4 +1,4 @@ -# Copyright 2020 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Template for .trampolinerc - # Add required env vars here. required_envvars+=( ) diff --git a/packages/pandas-gbq/MANIFEST.in b/packages/pandas-gbq/MANIFEST.in index e783f4c6209b..e0a66705318e 100644 --- a/packages/pandas-gbq/MANIFEST.in +++ b/packages/pandas-gbq/MANIFEST.in @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright 2020 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/docs/conf.py b/packages/pandas-gbq/docs/conf.py index ef544e748be5..3a9d090ea804 100644 --- a/packages/pandas-gbq/docs/conf.py +++ b/packages/pandas-gbq/docs/conf.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2021 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/scripts/decrypt-secrets.sh b/packages/pandas-gbq/scripts/decrypt-secrets.sh index 21f6d2a26d90..0018b421ddf8 100755 --- a/packages/pandas-gbq/scripts/decrypt-secrets.sh +++ b/packages/pandas-gbq/scripts/decrypt-secrets.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2015 Google Inc. All rights reserved. +# Copyright 2023 Google LLC All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/scripts/readme-gen/readme_gen.py b/packages/pandas-gbq/scripts/readme-gen/readme_gen.py index 91b59676bfc7..1acc119835b5 100644 --- a/packages/pandas-gbq/scripts/readme-gen/readme_gen.py +++ b/packages/pandas-gbq/scripts/readme-gen/readme_gen.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# Copyright 2016 Google Inc +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -33,17 +33,17 @@ autoescape=True, ) -README_TMPL = jinja_env.get_template('README.tmpl.rst') +README_TMPL = jinja_env.get_template("README.tmpl.rst") def get_help(file): - return subprocess.check_output(['python', file, '--help']).decode() + return subprocess.check_output(["python", file, "--help"]).decode() def main(): parser = argparse.ArgumentParser() - parser.add_argument('source') - parser.add_argument('--destination', default='README.rst') + parser.add_argument("source") + parser.add_argument("--destination", default="README.rst") args = parser.parse_args() @@ -51,9 +51,9 @@ def main(): root = os.path.dirname(source) destination = os.path.join(root, args.destination) - jinja_env.globals['get_help'] = get_help + jinja_env.globals["get_help"] = get_help - with io.open(source, 'r') as f: + with io.open(source, "r") as f: config = yaml.load(f) # This allows get_help to execute in the right directory. @@ -61,9 +61,9 @@ def main(): output = README_TMPL.render(config) - with io.open(destination, 'w') as f: + with io.open(destination, "w") as f: f.write(output) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/packages/pandas-gbq/setup.cfg b/packages/pandas-gbq/setup.cfg index c3a2b39f6528..052350089505 100644 --- a/packages/pandas-gbq/setup.cfg +++ b/packages/pandas-gbq/setup.cfg @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright 2020 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From bffde9b7bdf1666d9fe31abe4f840bda4b1d164e Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 21 Jul 2023 10:02:24 -0400 Subject: [PATCH 355/519] build(deps): [autoapprove] bump pygments from 2.13.0 to 2.15.0 (#656) Source-Link: https://github.com/googleapis/synthtool/commit/eaef28efd179e6eeb9f4e9bf697530d074a6f3b9 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:f8ca7655fa8a449cadcabcbce4054f593dcbae7aeeab34aa3fcc8b5cf7a93c9e Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 ++-- packages/pandas-gbq/.kokoro/requirements.txt | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index ae4a522b9e5f..17c21d96d654 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:39f0f3f2be02ef036e297e376fe3b6256775576da8a6ccb1d5eeb80f4c8bf8fb -# created: 2023-07-17T15:20:13.819193964Z + digest: sha256:f8ca7655fa8a449cadcabcbce4054f593dcbae7aeeab34aa3fcc8b5cf7a93c9e +# created: 2023-07-21T02:12:46.49799314Z diff --git a/packages/pandas-gbq/.kokoro/requirements.txt b/packages/pandas-gbq/.kokoro/requirements.txt index 67d70a110897..b563eb284459 100644 --- a/packages/pandas-gbq/.kokoro/requirements.txt +++ b/packages/pandas-gbq/.kokoro/requirements.txt @@ -396,9 +396,9 @@ pycparser==2.21 \ --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206 # via cffi -pygments==2.13.0 \ - --hash=sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1 \ - --hash=sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42 +pygments==2.15.0 \ + --hash=sha256:77a3299119af881904cd5ecd1ac6a66214b6e9bed1f2db16993b54adede64094 \ + --hash=sha256:f7e36cffc4c517fbc252861b9a6e4644ca0e5abadf9a113c72d1358ad09b9500 # via # readme-renderer # rich From b95b7dd2ba0e41eb75bcd975e3e8613eb94e15d9 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Sat, 22 Jul 2023 13:13:17 +0200 Subject: [PATCH 356/519] chore(deps): update dependency google-cloud-bigquery to v3.11.4 (#653) Co-authored-by: Anthonios Partheniou --- packages/pandas-gbq/samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index f01ccb1b7f20..3aace91bfa7f 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,5 +1,5 @@ google-cloud-bigquery-storage==2.22.0 -google-cloud-bigquery==3.11.3 +google-cloud-bigquery==3.11.4 pandas-gbq==0.19.2 pandas===1.3.5; python_version == '3.7' pandas==2.0.3; python_version >= '3.8' From 8a03f3799af0a3f1a99485ff4851105b053c40d2 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 27 Jul 2023 06:00:39 -0400 Subject: [PATCH 357/519] build(deps): [autoapprove] bump certifi from 2022.12.7 to 2023.7.22 (#658) Source-Link: https://github.com/googleapis/synthtool/commit/395d53adeeacfca00b73abf197f65f3c17c8f1e9 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:6c1cbc75c74b8bdd71dada2fa1677e9d6d78a889e9a70ee75b93d1d0543f96e1 Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 ++-- packages/pandas-gbq/.kokoro/requirements.txt | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 17c21d96d654..0ddd0e4d1873 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:f8ca7655fa8a449cadcabcbce4054f593dcbae7aeeab34aa3fcc8b5cf7a93c9e -# created: 2023-07-21T02:12:46.49799314Z + digest: sha256:6c1cbc75c74b8bdd71dada2fa1677e9d6d78a889e9a70ee75b93d1d0543f96e1 +# created: 2023-07-25T21:01:10.396410762Z diff --git a/packages/pandas-gbq/.kokoro/requirements.txt b/packages/pandas-gbq/.kokoro/requirements.txt index b563eb284459..76d9bba0f7d0 100644 --- a/packages/pandas-gbq/.kokoro/requirements.txt +++ b/packages/pandas-gbq/.kokoro/requirements.txt @@ -20,9 +20,9 @@ cachetools==5.2.0 \ --hash=sha256:6a94c6402995a99c3970cc7e4884bb60b4a8639938157eeed436098bf9831757 \ --hash=sha256:f9f17d2aec496a9aa6b76f53e3b614c965223c061982d434d160f930c698a9db # via google-auth -certifi==2022.12.7 \ - --hash=sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3 \ - --hash=sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18 +certifi==2023.7.22 \ + --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 \ + --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9 # via requests cffi==1.15.1 \ --hash=sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5 \ From bbc9c6089f9dbcab5b0ed2cc53db259e237b7698 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 1 Aug 2023 16:19:14 -0400 Subject: [PATCH 358/519] chore: [autoapprove] Pin flake8 version (#662) Source-Link: https://github.com/googleapis/synthtool/commit/0ddbff8012e47cde4462fe3f9feab01fbc4cdfd6 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:bced5ca77c4dda0fd2f5d845d4035fc3c5d3d6b81f245246a36aee114970082b Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 ++-- packages/pandas-gbq/.pre-commit-config.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 0ddd0e4d1873..d71329cc807d 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:6c1cbc75c74b8bdd71dada2fa1677e9d6d78a889e9a70ee75b93d1d0543f96e1 -# created: 2023-07-25T21:01:10.396410762Z + digest: sha256:bced5ca77c4dda0fd2f5d845d4035fc3c5d3d6b81f245246a36aee114970082b +# created: 2023-08-01T17:41:45.434027321Z diff --git a/packages/pandas-gbq/.pre-commit-config.yaml b/packages/pandas-gbq/.pre-commit-config.yaml index 9e3898fd1c12..19409cbd37a4 100644 --- a/packages/pandas-gbq/.pre-commit-config.yaml +++ b/packages/pandas-gbq/.pre-commit-config.yaml @@ -26,6 +26,6 @@ repos: hooks: - id: black - repo: https://github.com/pycqa/flake8 - rev: 3.9.2 + rev: 6.1.0 hooks: - id: flake8 From 7bcedce124eaa3818e65a5d987386ae05d442130 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 3 Aug 2023 10:42:42 -0400 Subject: [PATCH 359/519] build: [autoapprove] bump cryptography from 41.0.2 to 41.0.3 (#664) Source-Link: https://github.com/googleapis/synthtool/commit/352b9d4c068ce7c05908172af128b294073bf53c Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:3e3800bb100af5d7f9e810d48212b37812c1856d20ffeafb99ebe66461b61fc7 Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 +- packages/pandas-gbq/.kokoro/requirements.txt | 48 +++++++++---------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index d71329cc807d..a3da1b0d4cd3 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:bced5ca77c4dda0fd2f5d845d4035fc3c5d3d6b81f245246a36aee114970082b -# created: 2023-08-01T17:41:45.434027321Z + digest: sha256:3e3800bb100af5d7f9e810d48212b37812c1856d20ffeafb99ebe66461b61fc7 +# created: 2023-08-02T10:53:29.114535628Z diff --git a/packages/pandas-gbq/.kokoro/requirements.txt b/packages/pandas-gbq/.kokoro/requirements.txt index 76d9bba0f7d0..029bd342de94 100644 --- a/packages/pandas-gbq/.kokoro/requirements.txt +++ b/packages/pandas-gbq/.kokoro/requirements.txt @@ -113,30 +113,30 @@ commonmark==0.9.1 \ --hash=sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60 \ --hash=sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9 # via rich -cryptography==41.0.2 \ - --hash=sha256:01f1d9e537f9a15b037d5d9ee442b8c22e3ae11ce65ea1f3316a41c78756b711 \ - --hash=sha256:079347de771f9282fbfe0e0236c716686950c19dee1b76240ab09ce1624d76d7 \ - --hash=sha256:182be4171f9332b6741ee818ec27daff9fb00349f706629f5cbf417bd50e66fd \ - --hash=sha256:192255f539d7a89f2102d07d7375b1e0a81f7478925b3bc2e0549ebf739dae0e \ - --hash=sha256:2a034bf7d9ca894720f2ec1d8b7b5832d7e363571828037f9e0c4f18c1b58a58 \ - --hash=sha256:342f3767e25876751e14f8459ad85e77e660537ca0a066e10e75df9c9e9099f0 \ - --hash=sha256:439c3cc4c0d42fa999b83ded80a9a1fb54d53c58d6e59234cfe97f241e6c781d \ - --hash=sha256:49c3222bb8f8e800aead2e376cbef687bc9e3cb9b58b29a261210456a7783d83 \ - --hash=sha256:674b669d5daa64206c38e507808aae49904c988fa0a71c935e7006a3e1e83831 \ - --hash=sha256:7a9a3bced53b7f09da251685224d6a260c3cb291768f54954e28f03ef14e3766 \ - --hash=sha256:7af244b012711a26196450d34f483357e42aeddb04128885d95a69bd8b14b69b \ - --hash=sha256:7d230bf856164de164ecb615ccc14c7fc6de6906ddd5b491f3af90d3514c925c \ - --hash=sha256:84609ade00a6ec59a89729e87a503c6e36af98ddcd566d5f3be52e29ba993182 \ - --hash=sha256:9a6673c1828db6270b76b22cc696f40cde9043eb90373da5c2f8f2158957f42f \ - --hash=sha256:9b6d717393dbae53d4e52684ef4f022444fc1cce3c48c38cb74fca29e1f08eaa \ - --hash=sha256:9c3fe6534d59d071ee82081ca3d71eed3210f76ebd0361798c74abc2bcf347d4 \ - --hash=sha256:a719399b99377b218dac6cf547b6ec54e6ef20207b6165126a280b0ce97e0d2a \ - --hash=sha256:b332cba64d99a70c1e0836902720887fb4529ea49ea7f5462cf6640e095e11d2 \ - --hash=sha256:d124682c7a23c9764e54ca9ab5b308b14b18eba02722b8659fb238546de83a76 \ - --hash=sha256:d73f419a56d74fef257955f51b18d046f3506270a5fd2ac5febbfa259d6c0fa5 \ - --hash=sha256:f0dc40e6f7aa37af01aba07277d3d64d5a03dc66d682097541ec4da03cc140ee \ - --hash=sha256:f14ad275364c8b4e525d018f6716537ae7b6d369c094805cae45300847e0894f \ - --hash=sha256:f772610fe364372de33d76edcd313636a25684edb94cee53fd790195f5989d14 +cryptography==41.0.3 \ + --hash=sha256:0d09fb5356f975974dbcb595ad2d178305e5050656affb7890a1583f5e02a306 \ + --hash=sha256:23c2d778cf829f7d0ae180600b17e9fceea3c2ef8b31a99e3c694cbbf3a24b84 \ + --hash=sha256:3fb248989b6363906827284cd20cca63bb1a757e0a2864d4c1682a985e3dca47 \ + --hash=sha256:41d7aa7cdfded09b3d73a47f429c298e80796c8e825ddfadc84c8a7f12df212d \ + --hash=sha256:42cb413e01a5d36da9929baa9d70ca90d90b969269e5a12d39c1e0d475010116 \ + --hash=sha256:4c2f0d35703d61002a2bbdcf15548ebb701cfdd83cdc12471d2bae80878a4207 \ + --hash=sha256:4fd871184321100fb400d759ad0cddddf284c4b696568204d281c902fc7b0d81 \ + --hash=sha256:5259cb659aa43005eb55a0e4ff2c825ca111a0da1814202c64d28a985d33b087 \ + --hash=sha256:57a51b89f954f216a81c9d057bf1a24e2f36e764a1ca9a501a6964eb4a6800dd \ + --hash=sha256:652627a055cb52a84f8c448185922241dd5217443ca194d5739b44612c5e6507 \ + --hash=sha256:67e120e9a577c64fe1f611e53b30b3e69744e5910ff3b6e97e935aeb96005858 \ + --hash=sha256:6af1c6387c531cd364b72c28daa29232162010d952ceb7e5ca8e2827526aceae \ + --hash=sha256:6d192741113ef5e30d89dcb5b956ef4e1578f304708701b8b73d38e3e1461f34 \ + --hash=sha256:7efe8041897fe7a50863e51b77789b657a133c75c3b094e51b5e4b5cec7bf906 \ + --hash=sha256:84537453d57f55a50a5b6835622ee405816999a7113267739a1b4581f83535bd \ + --hash=sha256:8f09daa483aedea50d249ef98ed500569841d6498aa9c9f4b0531b9964658922 \ + --hash=sha256:95dd7f261bb76948b52a5330ba5202b91a26fbac13ad0e9fc8a3ac04752058c7 \ + --hash=sha256:a74fbcdb2a0d46fe00504f571a2a540532f4c188e6ccf26f1f178480117b33c4 \ + --hash=sha256:a983e441a00a9d57a4d7c91b3116a37ae602907a7618b882c8013b5762e80574 \ + --hash=sha256:ab8de0d091acbf778f74286f4989cf3d1528336af1b59f3e5d2ebca8b5fe49e1 \ + --hash=sha256:aeb57c421b34af8f9fe830e1955bf493a86a7996cc1338fe41b30047d16e962c \ + --hash=sha256:ce785cf81a7bdade534297ef9e490ddff800d956625020ab2ec2780a556c313e \ + --hash=sha256:d0d651aa754ef58d75cec6edfbd21259d93810b73f6ec246436a21b7841908de # via # gcp-releasetool # secretstorage From 9214b5eff704167b76ba83d90fb40cf7f0d5a6c3 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Wed, 16 Aug 2023 14:52:41 -0400 Subject: [PATCH 360/519] fix: edit units for time in tests (#655) * fix: edit units for time in tests * fix: updates timestamp precision for additional tests * fix: tweaking another time resolution * fix: tweaking yet another time resolution * trying out an option * minor syntax tweak, added comma * fix linting * test addition of dtypes to results * Update tests/system/test_gbq.py * Update tests/system/test_read_gbq.py * add tests to dtypes dict * Update tests/system/test_read_gbq.py * Update tests/system/test_gbq.py * try again * add version check and skipif on several tests * Update tests/system/test_read_gbq.py * blacken/lint the file * Adds a skipif statement to an additional test * blacken the code --- packages/pandas-gbq/tests/system/test_gbq.py | 4 ++++ packages/pandas-gbq/tests/system/test_read_gbq.py | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index 5b90e8ba5fb6..65c344a75e1d 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -5,6 +5,7 @@ # -*- coding: utf-8 -*- import datetime +import packaging.version import sys import numpy as np @@ -997,6 +998,9 @@ def test_upload_data_with_missing_schema_fields_raises_error(self, project_id): table_schema=test_schema, ) + @pytest.mark.skipif( + packaging.version.parse(pandas.__version__).release >= (2, 0), reason="" + ) def test_upload_data_with_timestamp(self, project_id): test_id = "21" test_size = 6 diff --git a/packages/pandas-gbq/tests/system/test_read_gbq.py b/packages/pandas-gbq/tests/system/test_read_gbq.py index f9358ef60bae..74cbfad92e37 100644 --- a/packages/pandas-gbq/tests/system/test_read_gbq.py +++ b/packages/pandas-gbq/tests/system/test_read_gbq.py @@ -5,6 +5,7 @@ import collections import datetime import decimal +import packaging.version import random import db_dtypes @@ -38,6 +39,9 @@ def writable_table( bigquery_client.delete_table(full_table_id) +@pytest.mark.skipif( + packaging.version.parse(pandas.__version__).release >= (2, 0), reason="" +) @pytest.mark.parametrize(["use_bqstorage_api"], [(True,), (False,)]) @pytest.mark.parametrize( ["query", "expected", "use_bqstorage_apis"], @@ -545,6 +549,7 @@ def writable_table( def test_default_dtypes( read_gbq, query, expected, use_bqstorage_apis, use_bqstorage_api ): + if use_bqstorage_api not in use_bqstorage_apis: pytest.skip(f"use_bqstorage_api={use_bqstorage_api} not supported.") # the parameter useQueryCache=False is used in the following function call From 2e54f57f300e4f1a2829f9669262448a4d0d0de2 Mon Sep 17 00:00:00 2001 From: meredithslota Date: Tue, 29 Aug 2023 06:26:41 -0700 Subject: [PATCH 361/519] chore: de-dupe contributing guides (#666) https://github.com/googleapis/python-bigquery-pandas/blob/main/CONTRIBUTING.rst is the correct contributing guide. --- packages/pandas-gbq/CONTRIBUTING.md | 35 ----------------------------- 1 file changed, 35 deletions(-) delete mode 100644 packages/pandas-gbq/CONTRIBUTING.md diff --git a/packages/pandas-gbq/CONTRIBUTING.md b/packages/pandas-gbq/CONTRIBUTING.md deleted file mode 100644 index a4f779149628..000000000000 --- a/packages/pandas-gbq/CONTRIBUTING.md +++ /dev/null @@ -1,35 +0,0 @@ -# Contributing - -We'd love to accept your patches and contributions to this project. There are -just a few small guidelines you need to follow. - -## Contributor License Agreement - -Contributions to this project must be accompanied by a Contributor License -Agreement (CLA). You (or your employer) retain the copyright to your -contribution; this simply gives us permission to use and redistribute your -contributions as part of the project. Head over to - to see your current agreements on file or -to sign a new one. - -You generally only need to submit a CLA once, so if you've already submitted one -(even if it was for a different project), you probably don't need to do it -again. - -## Code Reviews - -All submissions, including submissions by project members, require review. We -use GitHub pull requests for this purpose. Consult -[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more -information on using pull requests. - -## Community Guidelines - -This project follows -[Google's Open Source Community Guidelines](https://opensource.google/conduct/). - -## Developer Tips - -See the [contributing guide in the pandas-gbq -docs](http://pandas-gbq.readthedocs.io/en/latest/contributing.html). - From 1d7b817018d4cf223ffcbeb439ec114420e039d1 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 6 Oct 2023 22:01:11 -0400 Subject: [PATCH 362/519] chore: [autoapprove] bump cryptography from 41.0.3 to 41.0.4 (#676) Source-Link: https://github.com/googleapis/synthtool/commit/dede53ff326079b457cfb1aae5bbdc82cbb51dc3 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:fac304457974bb530cc5396abd4ab25d26a469cd3bc97cbfb18c8d4324c584eb Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 +- packages/pandas-gbq/.gitignore | 1 + packages/pandas-gbq/.kokoro/requirements.txt | 49 ++++++++++--------- 3 files changed, 28 insertions(+), 26 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index a3da1b0d4cd3..a9bdb1b7ac0f 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:3e3800bb100af5d7f9e810d48212b37812c1856d20ffeafb99ebe66461b61fc7 -# created: 2023-08-02T10:53:29.114535628Z + digest: sha256:fac304457974bb530cc5396abd4ab25d26a469cd3bc97cbfb18c8d4324c584eb +# created: 2023-10-02T21:31:03.517640371Z diff --git a/packages/pandas-gbq/.gitignore b/packages/pandas-gbq/.gitignore index b4243ced74e4..d083ea1ddc3e 100644 --- a/packages/pandas-gbq/.gitignore +++ b/packages/pandas-gbq/.gitignore @@ -50,6 +50,7 @@ docs.metadata # Virtual environment env/ +venv/ # Test logs coverage.xml diff --git a/packages/pandas-gbq/.kokoro/requirements.txt b/packages/pandas-gbq/.kokoro/requirements.txt index 029bd342de94..96d593c8c82a 100644 --- a/packages/pandas-gbq/.kokoro/requirements.txt +++ b/packages/pandas-gbq/.kokoro/requirements.txt @@ -113,30 +113,30 @@ commonmark==0.9.1 \ --hash=sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60 \ --hash=sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9 # via rich -cryptography==41.0.3 \ - --hash=sha256:0d09fb5356f975974dbcb595ad2d178305e5050656affb7890a1583f5e02a306 \ - --hash=sha256:23c2d778cf829f7d0ae180600b17e9fceea3c2ef8b31a99e3c694cbbf3a24b84 \ - --hash=sha256:3fb248989b6363906827284cd20cca63bb1a757e0a2864d4c1682a985e3dca47 \ - --hash=sha256:41d7aa7cdfded09b3d73a47f429c298e80796c8e825ddfadc84c8a7f12df212d \ - --hash=sha256:42cb413e01a5d36da9929baa9d70ca90d90b969269e5a12d39c1e0d475010116 \ - --hash=sha256:4c2f0d35703d61002a2bbdcf15548ebb701cfdd83cdc12471d2bae80878a4207 \ - --hash=sha256:4fd871184321100fb400d759ad0cddddf284c4b696568204d281c902fc7b0d81 \ - --hash=sha256:5259cb659aa43005eb55a0e4ff2c825ca111a0da1814202c64d28a985d33b087 \ - --hash=sha256:57a51b89f954f216a81c9d057bf1a24e2f36e764a1ca9a501a6964eb4a6800dd \ - --hash=sha256:652627a055cb52a84f8c448185922241dd5217443ca194d5739b44612c5e6507 \ - --hash=sha256:67e120e9a577c64fe1f611e53b30b3e69744e5910ff3b6e97e935aeb96005858 \ - --hash=sha256:6af1c6387c531cd364b72c28daa29232162010d952ceb7e5ca8e2827526aceae \ - --hash=sha256:6d192741113ef5e30d89dcb5b956ef4e1578f304708701b8b73d38e3e1461f34 \ - --hash=sha256:7efe8041897fe7a50863e51b77789b657a133c75c3b094e51b5e4b5cec7bf906 \ - --hash=sha256:84537453d57f55a50a5b6835622ee405816999a7113267739a1b4581f83535bd \ - --hash=sha256:8f09daa483aedea50d249ef98ed500569841d6498aa9c9f4b0531b9964658922 \ - --hash=sha256:95dd7f261bb76948b52a5330ba5202b91a26fbac13ad0e9fc8a3ac04752058c7 \ - --hash=sha256:a74fbcdb2a0d46fe00504f571a2a540532f4c188e6ccf26f1f178480117b33c4 \ - --hash=sha256:a983e441a00a9d57a4d7c91b3116a37ae602907a7618b882c8013b5762e80574 \ - --hash=sha256:ab8de0d091acbf778f74286f4989cf3d1528336af1b59f3e5d2ebca8b5fe49e1 \ - --hash=sha256:aeb57c421b34af8f9fe830e1955bf493a86a7996cc1338fe41b30047d16e962c \ - --hash=sha256:ce785cf81a7bdade534297ef9e490ddff800d956625020ab2ec2780a556c313e \ - --hash=sha256:d0d651aa754ef58d75cec6edfbd21259d93810b73f6ec246436a21b7841908de +cryptography==41.0.4 \ + --hash=sha256:004b6ccc95943f6a9ad3142cfabcc769d7ee38a3f60fb0dddbfb431f818c3a67 \ + --hash=sha256:047c4603aeb4bbd8db2756e38f5b8bd7e94318c047cfe4efeb5d715e08b49311 \ + --hash=sha256:0d9409894f495d465fe6fda92cb70e8323e9648af912d5b9141d616df40a87b8 \ + --hash=sha256:23a25c09dfd0d9f28da2352503b23e086f8e78096b9fd585d1d14eca01613e13 \ + --hash=sha256:2ed09183922d66c4ec5fdaa59b4d14e105c084dd0febd27452de8f6f74704143 \ + --hash=sha256:35c00f637cd0b9d5b6c6bd11b6c3359194a8eba9c46d4e875a3660e3b400005f \ + --hash=sha256:37480760ae08065437e6573d14be973112c9e6dcaf5f11d00147ee74f37a3829 \ + --hash=sha256:3b224890962a2d7b57cf5eeb16ccaafba6083f7b811829f00476309bce2fe0fd \ + --hash=sha256:5a0f09cefded00e648a127048119f77bc2b2ec61e736660b5789e638f43cc397 \ + --hash=sha256:5b72205a360f3b6176485a333256b9bcd48700fc755fef51c8e7e67c4b63e3ac \ + --hash=sha256:7e53db173370dea832190870e975a1e09c86a879b613948f09eb49324218c14d \ + --hash=sha256:7febc3094125fc126a7f6fb1f420d0da639f3f32cb15c8ff0dc3997c4549f51a \ + --hash=sha256:80907d3faa55dc5434a16579952ac6da800935cd98d14dbd62f6f042c7f5e839 \ + --hash=sha256:86defa8d248c3fa029da68ce61fe735432b047e32179883bdb1e79ed9bb8195e \ + --hash=sha256:8ac4f9ead4bbd0bc8ab2d318f97d85147167a488be0e08814a37eb2f439d5cf6 \ + --hash=sha256:93530900d14c37a46ce3d6c9e6fd35dbe5f5601bf6b3a5c325c7bffc030344d9 \ + --hash=sha256:9eeb77214afae972a00dee47382d2591abe77bdae166bda672fb1e24702a3860 \ + --hash=sha256:b5f4dfe950ff0479f1f00eda09c18798d4f49b98f4e2006d644b3301682ebdca \ + --hash=sha256:c3391bd8e6de35f6f1140e50aaeb3e2b3d6a9012536ca23ab0d9c35ec18c8a91 \ + --hash=sha256:c880eba5175f4307129784eca96f4e70b88e57aa3f680aeba3bab0e980b0f37d \ + --hash=sha256:cecfefa17042941f94ab54f769c8ce0fe14beff2694e9ac684176a2535bf9714 \ + --hash=sha256:e40211b4923ba5a6dc9769eab704bdb3fbb58d56c5b336d30996c24fcf12aadb \ + --hash=sha256:efc8ad4e6fc4f1752ebfb58aefece8b4e3c4cae940b0994d43649bdfce8d0d4f # via # gcp-releasetool # secretstorage @@ -382,6 +382,7 @@ protobuf==3.20.3 \ # gcp-docuploader # gcp-releasetool # google-api-core + # googleapis-common-protos pyasn1==0.4.8 \ --hash=sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d \ --hash=sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba From a0ac97fb942f6b062871ed20d0af20d8a2628841 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 10 Oct 2023 10:16:05 -0400 Subject: [PATCH 363/519] chore(deps): bump urllib3 from 1.26.12 to 1.26.17 in /.kokoro (#680) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: [autoapprove] Update `black` and `isort` to latest versions Source-Link: https://github.com/googleapis/synthtool/commit/0c7b0333f44b2b7075447f43a121a12d15a7b76a Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:08e34975760f002746b1d8c86fdc90660be45945ee6d9db914d1508acdf9a547 * Update black in noxfile.py * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot Co-authored-by: Anthonios Partheniou --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 ++-- packages/pandas-gbq/.kokoro/requirements.txt | 6 +++--- packages/pandas-gbq/.pre-commit-config.yaml | 2 +- packages/pandas-gbq/noxfile.py | 3 +-- packages/pandas-gbq/tests/system/test_read_gbq.py | 1 - 5 files changed, 7 insertions(+), 9 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index a9bdb1b7ac0f..dd98abbdeebe 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:fac304457974bb530cc5396abd4ab25d26a469cd3bc97cbfb18c8d4324c584eb -# created: 2023-10-02T21:31:03.517640371Z + digest: sha256:08e34975760f002746b1d8c86fdc90660be45945ee6d9db914d1508acdf9a547 +# created: 2023-10-09T14:06:13.397766266Z diff --git a/packages/pandas-gbq/.kokoro/requirements.txt b/packages/pandas-gbq/.kokoro/requirements.txt index 96d593c8c82a..0332d3267e15 100644 --- a/packages/pandas-gbq/.kokoro/requirements.txt +++ b/packages/pandas-gbq/.kokoro/requirements.txt @@ -467,9 +467,9 @@ typing-extensions==4.4.0 \ --hash=sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa \ --hash=sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e # via -r requirements.in -urllib3==1.26.12 \ - --hash=sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e \ - --hash=sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997 +urllib3==1.26.17 \ + --hash=sha256:24d6a242c28d29af46c3fae832c36db3bbebcc533dd1bb549172cd739c82df21 \ + --hash=sha256:94a757d178c9be92ef5539b8840d48dc9cf1b2709c9d6b588232a055c524458b # via # requests # twine diff --git a/packages/pandas-gbq/.pre-commit-config.yaml b/packages/pandas-gbq/.pre-commit-config.yaml index 19409cbd37a4..6a8e16950664 100644 --- a/packages/pandas-gbq/.pre-commit-config.yaml +++ b/packages/pandas-gbq/.pre-commit-config.yaml @@ -22,7 +22,7 @@ repos: - id: end-of-file-fixer - id: check-yaml - repo: https://github.com/psf/black - rev: 22.3.0 + rev: 23.7.0 hooks: - id: black - repo: https://github.com/pycqa/flake8 diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 135ce432c5cd..d491f4274b24 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -25,7 +25,7 @@ import nox -BLACK_VERSION = "black==22.3.0" +BLACK_VERSION = "black==23.7.0" ISORT_VERSION = "isort==5.10.1" LINT_PATHS = ["docs", "pandas_gbq", "tests", "noxfile.py", "setup.py"] @@ -200,7 +200,6 @@ def unit(session): def install_systemtest_dependencies(session, *constraints): - # Use pre-release gRPC for system tests. # Exclude version 1.52.0rc1 which has a known issue. # See https://github.com/grpc/grpc/issues/32163 diff --git a/packages/pandas-gbq/tests/system/test_read_gbq.py b/packages/pandas-gbq/tests/system/test_read_gbq.py index 74cbfad92e37..d57477b1148d 100644 --- a/packages/pandas-gbq/tests/system/test_read_gbq.py +++ b/packages/pandas-gbq/tests/system/test_read_gbq.py @@ -549,7 +549,6 @@ def writable_table( def test_default_dtypes( read_gbq, query, expected, use_bqstorage_apis, use_bqstorage_api ): - if use_bqstorage_api not in use_bqstorage_apis: pytest.skip(f"use_bqstorage_api={use_bqstorage_api} not supported.") # the parameter useQueryCache=False is used in the following function call From 08dacf9fefe7b9431d0dd3322fffad470601a494 Mon Sep 17 00:00:00 2001 From: Lingqing Gan Date: Mon, 23 Oct 2023 17:11:41 -0400 Subject: [PATCH 364/519] docs: migrate .readthedocs.yml to configuration file v2 (#689) * docs: set version: 2 in .readthedocs.yml * move requirement file to python.requirements * remove pip_install: true * change build.image to build.os * fix requirement docs name --- packages/pandas-gbq/.readthedocs.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/packages/pandas-gbq/.readthedocs.yml b/packages/pandas-gbq/.readthedocs.yml index 9b3d685401ea..7ae21e160a99 100644 --- a/packages/pandas-gbq/.readthedocs.yml +++ b/packages/pandas-gbq/.readthedocs.yml @@ -2,9 +2,17 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -requirements_file: docs/requirements-docs.txt +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +version: 2 + build: - image: latest + os: ubuntu-22.04 + tools: + python: "3.11" + python: - pip_install: true - version: 3.8 + install: + - requirements: docs/requirements-docs.txt From 871f80eae989d951ffa77c91609a575ade5d06e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Budzy=C5=84ski?= <37701616+pbudzyns@users.noreply.github.com> Date: Sat, 28 Oct 2023 01:45:24 +0200 Subject: [PATCH 365/519] doc: update authentication doc (#669) Co-authored-by: Lingqing Gan --- packages/pandas-gbq/docs/howto/authentication.rst | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/pandas-gbq/docs/howto/authentication.rst b/packages/pandas-gbq/docs/howto/authentication.rst index 17e6c04a305d..2af4801ff117 100644 --- a/packages/pandas-gbq/docs/howto/authentication.rst +++ b/packages/pandas-gbq/docs/howto/authentication.rst @@ -10,7 +10,7 @@ Before you begin, you must create a Google Cloud Platform project. Use the the service for free. pandas-gbq `authenticates with the Google BigQuery service -`_ via OAuth 2.0. Use +`_ via OAuth 2.0. Use the ``credentials`` argument to explicitly pass in Google :class:`~google.auth.credentials.Credentials`. @@ -132,6 +132,13 @@ To use service account credentials, set the ``credentials`` parameter to the res ) df = pandas_gbq.read_gbq(sql, project_id="YOUR-PROJECT-ID", credentials=credentials) +Alternatively, you can set ``GOOGLE_APPLICATION_CREDENTIALS`` environment variable to the +full path to the JSON file. + +.. code-block:: shell + + $ export GOOGLE_APPLICATION_CREDENTIALS=/path/to/key.json + Use the :func:`~google.oauth2.service_account.Credentials.with_scopes` method to use authorize with specific OAuth2 scopes, which may be required in queries to federated data sources such as Google Sheets. @@ -148,8 +155,10 @@ queries to federated data sources such as Google Sheets. df = pandas_gbq.read_gbq(..., credentials=credentials) See the `Getting started with authentication on Google Cloud Platform -`_ guide for -more information on service accounts. +`_ guide and +`Google Auth Library User Guide +`_ for more information +on service accounts. .. _authentication-user: From fdd8b7749e21bc487188ef045235472355986b25 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Mon, 30 Oct 2023 07:27:44 -0400 Subject: [PATCH 366/519] chore: rename rst files to avoid conflict with service names (#688) Source-Link: https://github.com/googleapis/synthtool/commit/d52e638b37b091054c869bfa6f5a9fedaba9e0dd Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:4f9b3b106ad0beafc2c8a415e3f62c1a0cc23cabea115dbe841b848f581cfe99 Co-authored-by: Owl Bot Co-authored-by: Victor Chudnovsky --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 ++-- packages/pandas-gbq/.kokoro/requirements.txt | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index dd98abbdeebe..7f291dbd5f9b 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:08e34975760f002746b1d8c86fdc90660be45945ee6d9db914d1508acdf9a547 -# created: 2023-10-09T14:06:13.397766266Z + digest: sha256:4f9b3b106ad0beafc2c8a415e3f62c1a0cc23cabea115dbe841b848f581cfe99 +# created: 2023-10-18T20:26:37.410353675Z diff --git a/packages/pandas-gbq/.kokoro/requirements.txt b/packages/pandas-gbq/.kokoro/requirements.txt index 0332d3267e15..16170d0ca7b8 100644 --- a/packages/pandas-gbq/.kokoro/requirements.txt +++ b/packages/pandas-gbq/.kokoro/requirements.txt @@ -467,9 +467,9 @@ typing-extensions==4.4.0 \ --hash=sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa \ --hash=sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e # via -r requirements.in -urllib3==1.26.17 \ - --hash=sha256:24d6a242c28d29af46c3fae832c36db3bbebcc533dd1bb549172cd739c82df21 \ - --hash=sha256:94a757d178c9be92ef5539b8840d48dc9cf1b2709c9d6b588232a055c524458b +urllib3==1.26.18 \ + --hash=sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07 \ + --hash=sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0 # via # requests # twine From c12fce277a4417f034375d65e05766f12aae5507 Mon Sep 17 00:00:00 2001 From: Lingqing Gan Date: Wed, 1 Nov 2023 14:17:50 -0400 Subject: [PATCH 367/519] fix: table schema change error (#692) * fix: table schema change error * improve comments --- packages/pandas-gbq/pandas_gbq/gbq.py | 15 +++--- packages/pandas-gbq/tests/system/test_gbq.py | 14 +++--- packages/pandas-gbq/tests/unit/test_gbq.py | 53 ++++++++++++++++++++ 3 files changed, 70 insertions(+), 12 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index edf827343120..26a566d9c414 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -1205,12 +1205,15 @@ def to_gbq( ) table_connector.create(table_id, table_schema) else: - # Convert original schema (the schema that already exists) to pandas-gbq API format - original_schema = pandas_gbq.schema.to_pandas_gbq(table.schema) - - # Update the local `table_schema` so mode (NULLABLE/REQUIRED) - # matches. See: https://github.com/pydata/pandas-gbq/issues/315 - table_schema = pandas_gbq.schema.update_schema(table_schema, original_schema) + if if_exists == "append": + # Convert original schema (the schema that already exists) to pandas-gbq API format + original_schema = pandas_gbq.schema.to_pandas_gbq(table.schema) + + # Update the local `table_schema` so mode (NULLABLE/REQUIRED) + # matches. See: https://github.com/pydata/pandas-gbq/issues/315 + table_schema = pandas_gbq.schema.update_schema( + table_schema, original_schema + ) if dataframe.empty: # Create the table (if needed), but don't try to run a load job with an diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index 65c344a75e1d..9aac2357e651 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -788,6 +788,7 @@ def test_upload_data_if_table_exists_replace(self, project_id): test_size = 10 df = make_mixed_dataframe_v2(test_size) df_different_schema = make_mixed_dataframe_v1() + schema_new = gbq.generate_bq_schema(df_different_schema) # Initialize table with sample data gbq.to_gbq( @@ -798,7 +799,7 @@ def test_upload_data_if_table_exists_replace(self, project_id): credentials=self.credentials, ) - # Test the if_exists parameter with the value 'replace'. + # When if_exists == 'replace', table schema should change too. gbq.to_gbq( df_different_schema, self.destination_table + test_id, @@ -807,15 +808,16 @@ def test_upload_data_if_table_exists_replace(self, project_id): credentials=self.credentials, ) - result = gbq.read_gbq( - "SELECT COUNT(*) AS num_rows FROM {0}".format( - self.destination_table + test_id - ), + df_new = gbq.read_gbq( + "SELECT * FROM {0}".format(self.destination_table + test_id), project_id=project_id, credentials=self.credentials, dialect="legacy", ) - assert result["num_rows"][0] == 5 + + schema_returned = gbq.generate_bq_schema(df_new) + assert schema_new == schema_returned + assert df_new.shape[0] == 5 def test_upload_data_if_table_exists_raises_value_error(self, project_id): test_id = "4" diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index 5184562ab293..ba6206865998 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -650,6 +650,59 @@ def test_load_does_not_modify_schema_arg(mock_bigquery_client): assert original_schema == original_schema_cp +def test_load_modifies_schema(mock_bigquery_client): + """Test of https://github.com/googleapis/python-bigquery-pandas/issues/670""" + from google.api_core.exceptions import NotFound + + # Create table with new schema. + mock_bigquery_client.get_table.side_effect = NotFound("nope") + df = DataFrame( + { + "field1": ["a", "b"], + "field2": [1, 2], + "field3": [datetime.date(2019, 1, 1), datetime.date(2019, 5, 1)], + } + ) + original_schema = [ + {"name": "field1", "type": "STRING", "mode": "REQUIRED"}, + {"name": "field2", "type": "INTEGER"}, + {"name": "field3", "type": "DATE"}, + ] + original_schema_cp = copy.deepcopy(original_schema) + gbq.to_gbq( + df, + "dataset.schematest", + project_id="my-project", + table_schema=original_schema, + if_exists="fail", + ) + assert original_schema == original_schema_cp + + # Test that when if_exists == "replace", the new table schema updates + # according to the local schema. + new_df = DataFrame( + { + "field1": ["a", "b"], + "field2": ["c", "d"], + "field3": [datetime.date(2019, 1, 1), datetime.date(2019, 5, 1)], + } + ) + new_schema = [ + {"name": "field1", "type": "STRING", "mode": "REQUIRED"}, + {"name": "field2", "type": "STRING"}, + {"name": "field3", "type": "DATE"}, + ] + new_schema_cp = copy.deepcopy(new_schema) + gbq.to_gbq( + new_df, + "dataset.schematest", + project_id="my-project", + table_schema=new_schema, + if_exists="replace", + ) + assert new_schema == new_schema_cp + + def test_read_gbq_passes_dtypes(mock_bigquery_client, mock_service_account_credentials): mock_service_account_credentials.project_id = "service_account_project_id" df = gbq.read_gbq( From 0d75d32dce8ac9c682febe0b3079525b3280d012 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 2 Nov 2023 21:50:08 -0400 Subject: [PATCH 368/519] chore: update docfx minimum Python version (#693) * chore: update docfx minimum Python version Source-Link: https://github.com/googleapis/synthtool/commit/bc07fd415c39853b382bcf8315f8eeacdf334055 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:30470597773378105e239b59fce8eb27cc97375580d592699206d17d117143d0 * update docfx nox session to use python 3.10 --------- Co-authored-by: Owl Bot Co-authored-by: Anthonios Partheniou --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 ++-- packages/pandas-gbq/.github/workflows/docs.yml | 2 +- packages/pandas-gbq/noxfile.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 7f291dbd5f9b..ec696b558c35 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:4f9b3b106ad0beafc2c8a415e3f62c1a0cc23cabea115dbe841b848f581cfe99 -# created: 2023-10-18T20:26:37.410353675Z + digest: sha256:30470597773378105e239b59fce8eb27cc97375580d592699206d17d117143d0 +# created: 2023-11-03T00:57:07.335914631Z diff --git a/packages/pandas-gbq/.github/workflows/docs.yml b/packages/pandas-gbq/.github/workflows/docs.yml index e97d89e484c9..221806cedf58 100644 --- a/packages/pandas-gbq/.github/workflows/docs.yml +++ b/packages/pandas-gbq/.github/workflows/docs.yml @@ -28,7 +28,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: "3.9" + python-version: "3.10" - name: Install nox run: | python -m pip install --upgrade setuptools pip wheel diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index d491f4274b24..051f5da4dc00 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -395,7 +395,7 @@ def docs(session): ) -@nox.session(python="3.9") +@nox.session(python="3.10") def docfx(session): """Build the docfx yaml files for this library.""" From f50e69aea6951a603d8af0092ed605c2725a586c Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Wed, 8 Nov 2023 12:59:29 -0800 Subject: [PATCH 369/519] chore: bump urllib3 from 1.26.12 to 1.26.18 (#695) Source-Link: https://github.com/googleapis/synthtool/commit/febacccc98d6d224aff9d0bd0373bb5a4cd5969c Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:caffe0a9277daeccc4d1de5c9b55ebba0901b57c2f713ec9c876b0d4ec064f61 Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 +- packages/pandas-gbq/.kokoro/requirements.txt | 532 +++++++++--------- 2 files changed, 277 insertions(+), 259 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index ec696b558c35..453b540c1e58 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:30470597773378105e239b59fce8eb27cc97375580d592699206d17d117143d0 -# created: 2023-11-03T00:57:07.335914631Z + digest: sha256:caffe0a9277daeccc4d1de5c9b55ebba0901b57c2f713ec9c876b0d4ec064f61 +# created: 2023-11-08T19:46:45.022803742Z diff --git a/packages/pandas-gbq/.kokoro/requirements.txt b/packages/pandas-gbq/.kokoro/requirements.txt index 16170d0ca7b8..8957e21104e2 100644 --- a/packages/pandas-gbq/.kokoro/requirements.txt +++ b/packages/pandas-gbq/.kokoro/requirements.txt @@ -4,91 +4,75 @@ # # pip-compile --allow-unsafe --generate-hashes requirements.in # -argcomplete==2.0.0 \ - --hash=sha256:6372ad78c89d662035101418ae253668445b391755cfe94ea52f1b9d22425b20 \ - --hash=sha256:cffa11ea77999bb0dd27bb25ff6dc142a6796142f68d45b1a26b11f58724561e +argcomplete==3.1.4 \ + --hash=sha256:72558ba729e4c468572609817226fb0a6e7e9a0a7d477b882be168c0b4a62b94 \ + --hash=sha256:fbe56f8cda08aa9a04b307d8482ea703e96a6a801611acb4be9bf3942017989f # via nox -attrs==22.1.0 \ - --hash=sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6 \ - --hash=sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c +attrs==23.1.0 \ + --hash=sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04 \ + --hash=sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015 # via gcp-releasetool -bleach==5.0.1 \ - --hash=sha256:085f7f33c15bd408dd9b17a4ad77c577db66d76203e5984b1bd59baeee948b2a \ - --hash=sha256:0d03255c47eb9bd2f26aa9bb7f2107732e7e8fe195ca2f64709fcf3b0a4a085c - # via readme-renderer -cachetools==5.2.0 \ - --hash=sha256:6a94c6402995a99c3970cc7e4884bb60b4a8639938157eeed436098bf9831757 \ - --hash=sha256:f9f17d2aec496a9aa6b76f53e3b614c965223c061982d434d160f930c698a9db +cachetools==5.3.2 \ + --hash=sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2 \ + --hash=sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1 # via google-auth certifi==2023.7.22 \ --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 \ --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9 # via requests -cffi==1.15.1 \ - --hash=sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5 \ - --hash=sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef \ - --hash=sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104 \ - --hash=sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426 \ - --hash=sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405 \ - --hash=sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375 \ - --hash=sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a \ - --hash=sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e \ - --hash=sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc \ - --hash=sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf \ - --hash=sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185 \ - --hash=sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497 \ - --hash=sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3 \ - --hash=sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35 \ - --hash=sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c \ - --hash=sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83 \ - --hash=sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21 \ - --hash=sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca \ - --hash=sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984 \ - --hash=sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac \ - --hash=sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd \ - --hash=sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee \ - --hash=sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a \ - --hash=sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2 \ - --hash=sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192 \ - --hash=sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7 \ - --hash=sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585 \ - --hash=sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f \ - --hash=sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e \ - --hash=sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27 \ - --hash=sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b \ - --hash=sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e \ - --hash=sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e \ - --hash=sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d \ - --hash=sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c \ - --hash=sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415 \ - --hash=sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82 \ - --hash=sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02 \ - --hash=sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314 \ - --hash=sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325 \ - --hash=sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c \ - --hash=sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3 \ - --hash=sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914 \ - --hash=sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045 \ - --hash=sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d \ - --hash=sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9 \ - --hash=sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5 \ - --hash=sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2 \ - --hash=sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c \ - --hash=sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3 \ - --hash=sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2 \ - --hash=sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8 \ - --hash=sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d \ - --hash=sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d \ - --hash=sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9 \ - --hash=sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162 \ - --hash=sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76 \ - --hash=sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4 \ - --hash=sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e \ - --hash=sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9 \ - --hash=sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6 \ - --hash=sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b \ - --hash=sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01 \ - --hash=sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0 +cffi==1.16.0 \ + --hash=sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc \ + --hash=sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a \ + --hash=sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417 \ + --hash=sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab \ + --hash=sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520 \ + --hash=sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36 \ + --hash=sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743 \ + --hash=sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8 \ + --hash=sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed \ + --hash=sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684 \ + --hash=sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56 \ + --hash=sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324 \ + --hash=sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d \ + --hash=sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235 \ + --hash=sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e \ + --hash=sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088 \ + --hash=sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000 \ + --hash=sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7 \ + --hash=sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e \ + --hash=sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673 \ + --hash=sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c \ + --hash=sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe \ + --hash=sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2 \ + --hash=sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098 \ + --hash=sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8 \ + --hash=sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a \ + --hash=sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0 \ + --hash=sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b \ + --hash=sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896 \ + --hash=sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e \ + --hash=sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9 \ + --hash=sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2 \ + --hash=sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b \ + --hash=sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6 \ + --hash=sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404 \ + --hash=sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f \ + --hash=sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0 \ + --hash=sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4 \ + --hash=sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc \ + --hash=sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936 \ + --hash=sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba \ + --hash=sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872 \ + --hash=sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb \ + --hash=sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614 \ + --hash=sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1 \ + --hash=sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d \ + --hash=sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969 \ + --hash=sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b \ + --hash=sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4 \ + --hash=sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627 \ + --hash=sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956 \ + --hash=sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357 # via cryptography charset-normalizer==2.1.1 \ --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 \ @@ -109,78 +93,74 @@ colorlog==6.7.0 \ # via # gcp-docuploader # nox -commonmark==0.9.1 \ - --hash=sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60 \ - --hash=sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9 - # via rich -cryptography==41.0.4 \ - --hash=sha256:004b6ccc95943f6a9ad3142cfabcc769d7ee38a3f60fb0dddbfb431f818c3a67 \ - --hash=sha256:047c4603aeb4bbd8db2756e38f5b8bd7e94318c047cfe4efeb5d715e08b49311 \ - --hash=sha256:0d9409894f495d465fe6fda92cb70e8323e9648af912d5b9141d616df40a87b8 \ - --hash=sha256:23a25c09dfd0d9f28da2352503b23e086f8e78096b9fd585d1d14eca01613e13 \ - --hash=sha256:2ed09183922d66c4ec5fdaa59b4d14e105c084dd0febd27452de8f6f74704143 \ - --hash=sha256:35c00f637cd0b9d5b6c6bd11b6c3359194a8eba9c46d4e875a3660e3b400005f \ - --hash=sha256:37480760ae08065437e6573d14be973112c9e6dcaf5f11d00147ee74f37a3829 \ - --hash=sha256:3b224890962a2d7b57cf5eeb16ccaafba6083f7b811829f00476309bce2fe0fd \ - --hash=sha256:5a0f09cefded00e648a127048119f77bc2b2ec61e736660b5789e638f43cc397 \ - --hash=sha256:5b72205a360f3b6176485a333256b9bcd48700fc755fef51c8e7e67c4b63e3ac \ - --hash=sha256:7e53db173370dea832190870e975a1e09c86a879b613948f09eb49324218c14d \ - --hash=sha256:7febc3094125fc126a7f6fb1f420d0da639f3f32cb15c8ff0dc3997c4549f51a \ - --hash=sha256:80907d3faa55dc5434a16579952ac6da800935cd98d14dbd62f6f042c7f5e839 \ - --hash=sha256:86defa8d248c3fa029da68ce61fe735432b047e32179883bdb1e79ed9bb8195e \ - --hash=sha256:8ac4f9ead4bbd0bc8ab2d318f97d85147167a488be0e08814a37eb2f439d5cf6 \ - --hash=sha256:93530900d14c37a46ce3d6c9e6fd35dbe5f5601bf6b3a5c325c7bffc030344d9 \ - --hash=sha256:9eeb77214afae972a00dee47382d2591abe77bdae166bda672fb1e24702a3860 \ - --hash=sha256:b5f4dfe950ff0479f1f00eda09c18798d4f49b98f4e2006d644b3301682ebdca \ - --hash=sha256:c3391bd8e6de35f6f1140e50aaeb3e2b3d6a9012536ca23ab0d9c35ec18c8a91 \ - --hash=sha256:c880eba5175f4307129784eca96f4e70b88e57aa3f680aeba3bab0e980b0f37d \ - --hash=sha256:cecfefa17042941f94ab54f769c8ce0fe14beff2694e9ac684176a2535bf9714 \ - --hash=sha256:e40211b4923ba5a6dc9769eab704bdb3fbb58d56c5b336d30996c24fcf12aadb \ - --hash=sha256:efc8ad4e6fc4f1752ebfb58aefece8b4e3c4cae940b0994d43649bdfce8d0d4f +cryptography==41.0.5 \ + --hash=sha256:0c327cac00f082013c7c9fb6c46b7cc9fa3c288ca702c74773968173bda421bf \ + --hash=sha256:0d2a6a598847c46e3e321a7aef8af1436f11c27f1254933746304ff014664d84 \ + --hash=sha256:227ec057cd32a41c6651701abc0328135e472ed450f47c2766f23267b792a88e \ + --hash=sha256:22892cc830d8b2c89ea60148227631bb96a7da0c1b722f2aac8824b1b7c0b6b8 \ + --hash=sha256:392cb88b597247177172e02da6b7a63deeff1937fa6fec3bbf902ebd75d97ec7 \ + --hash=sha256:3be3ca726e1572517d2bef99a818378bbcf7d7799d5372a46c79c29eb8d166c1 \ + --hash=sha256:573eb7128cbca75f9157dcde974781209463ce56b5804983e11a1c462f0f4e88 \ + --hash=sha256:580afc7b7216deeb87a098ef0674d6ee34ab55993140838b14c9b83312b37b86 \ + --hash=sha256:5a70187954ba7292c7876734183e810b728b4f3965fbe571421cb2434d279179 \ + --hash=sha256:73801ac9736741f220e20435f84ecec75ed70eda90f781a148f1bad546963d81 \ + --hash=sha256:7d208c21e47940369accfc9e85f0de7693d9a5d843c2509b3846b2db170dfd20 \ + --hash=sha256:8254962e6ba1f4d2090c44daf50a547cd5f0bf446dc658a8e5f8156cae0d8548 \ + --hash=sha256:88417bff20162f635f24f849ab182b092697922088b477a7abd6664ddd82291d \ + --hash=sha256:a48e74dad1fb349f3dc1d449ed88e0017d792997a7ad2ec9587ed17405667e6d \ + --hash=sha256:b948e09fe5fb18517d99994184854ebd50b57248736fd4c720ad540560174ec5 \ + --hash=sha256:c707f7afd813478e2019ae32a7c49cd932dd60ab2d2a93e796f68236b7e1fbf1 \ + --hash=sha256:d38e6031e113b7421db1de0c1b1f7739564a88f1684c6b89234fbf6c11b75147 \ + --hash=sha256:d3977f0e276f6f5bf245c403156673db103283266601405376f075c849a0b936 \ + --hash=sha256:da6a0ff8f1016ccc7477e6339e1d50ce5f59b88905585f77193ebd5068f1e797 \ + --hash=sha256:e270c04f4d9b5671ebcc792b3ba5d4488bf7c42c3c241a3748e2599776f29696 \ + --hash=sha256:e886098619d3815e0ad5790c973afeee2c0e6e04b4da90b88e6bd06e2a0b1b72 \ + --hash=sha256:ec3b055ff8f1dce8e6ef28f626e0972981475173d7973d63f271b29c8a2897da \ + --hash=sha256:fba1e91467c65fe64a82c689dc6cf58151158993b13eb7a7f3f4b7f395636723 # via # gcp-releasetool # secretstorage -distlib==0.3.6 \ - --hash=sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46 \ - --hash=sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e +distlib==0.3.7 \ + --hash=sha256:2e24928bc811348f0feb63014e97aaae3037f2cf48712d51ae61df7fd6075057 \ + --hash=sha256:9dafe54b34a028eafd95039d5e5d4851a13734540f1331060d31c9916e7147a8 # via virtualenv -docutils==0.19 \ - --hash=sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6 \ - --hash=sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc +docutils==0.20.1 \ + --hash=sha256:96f387a2c5562db4476f09f13bbab2192e764cac08ebbf3a34a95d9b1e4a59d6 \ + --hash=sha256:f08a4e276c3a1583a86dce3e34aba3fe04d02bba2dd51ed16106244e8a923e3b # via readme-renderer -filelock==3.8.0 \ - --hash=sha256:55447caa666f2198c5b6b13a26d2084d26fa5b115c00d065664b2124680c4edc \ - --hash=sha256:617eb4e5eedc82fc5f47b6d61e4d11cb837c56cb4544e39081099fa17ad109d4 +filelock==3.13.1 \ + --hash=sha256:521f5f56c50f8426f5e03ad3b281b490a87ef15bc6c526f168290f0c7148d44e \ + --hash=sha256:57dbda9b35157b05fb3e58ee91448612eb674172fab98ee235ccb0b5bee19a1c # via virtualenv -gcp-docuploader==0.6.4 \ - --hash=sha256:01486419e24633af78fd0167db74a2763974765ee8078ca6eb6964d0ebd388af \ - --hash=sha256:70861190c123d907b3b067da896265ead2eeb9263969d6955c9e0bb091b5ccbf +gcp-docuploader==0.6.5 \ + --hash=sha256:30221d4ac3e5a2b9c69aa52fdbef68cc3f27d0e6d0d90e220fc024584b8d2318 \ + --hash=sha256:b7458ef93f605b9d46a4bf3a8dc1755dad1f31d030c8679edf304e343b347eea # via -r requirements.in -gcp-releasetool==1.10.5 \ - --hash=sha256:174b7b102d704b254f2a26a3eda2c684fd3543320ec239baf771542a2e58e109 \ - --hash=sha256:e29d29927fe2ca493105a82958c6873bb2b90d503acac56be2c229e74de0eec9 +gcp-releasetool==1.16.0 \ + --hash=sha256:27bf19d2e87aaa884096ff941aa3c592c482be3d6a2bfe6f06afafa6af2353e3 \ + --hash=sha256:a316b197a543fd036209d0caba7a8eb4d236d8e65381c80cbc6d7efaa7606d63 # via -r requirements.in -google-api-core==2.10.2 \ - --hash=sha256:10c06f7739fe57781f87523375e8e1a3a4674bf6392cd6131a3222182b971320 \ - --hash=sha256:34f24bd1d5f72a8c4519773d99ca6bf080a6c4e041b4e9f024fe230191dda62e +google-api-core==2.12.0 \ + --hash=sha256:c22e01b1e3c4dcd90998494879612c38d0a3411d1f7b679eb89e2abe3ce1f553 \ + --hash=sha256:ec6054f7d64ad13b41e43d96f735acbd763b0f3b695dabaa2d579673f6a6e160 # via # google-cloud-core # google-cloud-storage -google-auth==2.14.1 \ - --hash=sha256:ccaa901f31ad5cbb562615eb8b664b3dd0bf5404a67618e642307f00613eda4d \ - --hash=sha256:f5d8701633bebc12e0deea4df8abd8aff31c28b355360597f7f2ee60f2e4d016 +google-auth==2.23.4 \ + --hash=sha256:79905d6b1652187def79d491d6e23d0cbb3a21d3c7ba0dbaa9c8a01906b13ff3 \ + --hash=sha256:d4bbc92fe4b8bfd2f3e8d88e5ba7085935da208ee38a134fc280e7ce682a05f2 # via # gcp-releasetool # google-api-core # google-cloud-core # google-cloud-storage -google-cloud-core==2.3.2 \ - --hash=sha256:8417acf6466be2fa85123441696c4badda48db314c607cf1e5d543fa8bdc22fe \ - --hash=sha256:b9529ee7047fd8d4bf4a2182de619154240df17fbe60ead399078c1ae152af9a +google-cloud-core==2.3.3 \ + --hash=sha256:37b80273c8d7eee1ae816b3a20ae43585ea50506cb0e60f3cf5be5f87f1373cb \ + --hash=sha256:fbd11cad3e98a7e5b0343dc07cb1039a5ffd7a5bb96e1f1e27cee4bda4a90863 # via google-cloud-storage -google-cloud-storage==2.6.0 \ - --hash=sha256:104ca28ae61243b637f2f01455cc8a05e8f15a2a18ced96cb587241cdd3820f5 \ - --hash=sha256:4ad0415ff61abdd8bb2ae81c1f8f7ec7d91a1011613f2db87c614c550f97bfe9 +google-cloud-storage==2.13.0 \ + --hash=sha256:ab0bf2e1780a1b74cf17fccb13788070b729f50c252f0c94ada2aae0ca95437d \ + --hash=sha256:f62dc4c7b6cd4360d072e3deb28035fbdad491ac3d9b0b1815a12daea10f37c7 # via gcp-docuploader google-crc32c==1.5.0 \ --hash=sha256:024894d9d3cfbc5943f8f230e23950cd4906b2fe004c72e29b209420a1e6b05a \ @@ -251,29 +231,31 @@ google-crc32c==1.5.0 \ --hash=sha256:f583edb943cf2e09c60441b910d6a20b4d9d626c75a36c8fcac01a6c96c01183 \ --hash=sha256:fd8536e902db7e365f49e7d9029283403974ccf29b13fc7028b97e2295b33556 \ --hash=sha256:fe70e325aa68fa4b5edf7d1a4b6f691eb04bbccac0ace68e34820d283b5f80d4 - # via google-resumable-media -google-resumable-media==2.4.0 \ - --hash=sha256:2aa004c16d295c8f6c33b2b4788ba59d366677c0a25ae7382436cb30f776deaa \ - --hash=sha256:8d5518502f92b9ecc84ac46779bd4f09694ecb3ba38a3e7ca737a86d15cbca1f + # via + # google-cloud-storage + # google-resumable-media +google-resumable-media==2.6.0 \ + --hash=sha256:972852f6c65f933e15a4a210c2b96930763b47197cdf4aa5f5bea435efb626e7 \ + --hash=sha256:fc03d344381970f79eebb632a3c18bb1828593a2dc5572b5f90115ef7d11e81b # via google-cloud-storage -googleapis-common-protos==1.57.0 \ - --hash=sha256:27a849d6205838fb6cc3c1c21cb9800707a661bb21c6ce7fb13e99eb1f8a0c46 \ - --hash=sha256:a9f4a1d7f6d9809657b7f1316a1aa527f6664891531bcfcc13b6696e685f443c +googleapis-common-protos==1.61.0 \ + --hash=sha256:22f1915393bb3245343f6efe87f6fe868532efc12aa26b391b15132e1279f1c0 \ + --hash=sha256:8a64866a97f6304a7179873a465d6eee97b7a24ec6cfd78e0f575e96b821240b # via google-api-core idna==3.4 \ --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 \ --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2 # via requests -importlib-metadata==5.0.0 \ - --hash=sha256:da31db32b304314d044d3c12c79bd59e307889b287ad12ff387b3500835fc2ab \ - --hash=sha256:ddb0e35065e8938f867ed4928d0ae5bf2a53b7773871bfe6bcc7e4fcdc7dea43 +importlib-metadata==6.8.0 \ + --hash=sha256:3ebb78df84a805d7698245025b975d9d67053cd94c79245ba4b3eb694abe68bb \ + --hash=sha256:dbace7892d8c0c4ac1ad096662232f831d4e64f4c4545bd53016a3e9d4654743 # via # -r requirements.in # keyring # twine -jaraco-classes==3.2.3 \ - --hash=sha256:2353de3288bc6b82120752201c6b1c1a14b058267fa424ed5ce5984e3b922158 \ - --hash=sha256:89559fa5c1d3c34eff6f631ad80bb21f378dbcbb35dd161fd2c6b93f5be2f98a +jaraco-classes==3.3.0 \ + --hash=sha256:10afa92b6743f25c0cf5f37c6bb6e18e2c5bb84a16527ccfc0040ea377e7aaeb \ + --hash=sha256:c063dd08e89217cee02c8d5e5ec560f2c8ce6cdc2fcdc2e68f7b2e5547ed3621 # via keyring jeepney==0.8.0 \ --hash=sha256:5efe48d255973902f6badc3ce55e2aa6c5c3b3bc642059ef3a91247bcfcc5806 \ @@ -285,75 +267,121 @@ jinja2==3.1.2 \ --hash=sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852 \ --hash=sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61 # via gcp-releasetool -keyring==23.11.0 \ - --hash=sha256:3dd30011d555f1345dec2c262f0153f2f0ca6bca041fb1dc4588349bb4c0ac1e \ - --hash=sha256:ad192263e2cdd5f12875dedc2da13534359a7e760e77f8d04b50968a821c2361 +keyring==24.2.0 \ + --hash=sha256:4901caaf597bfd3bbd78c9a0c7c4c29fcd8310dab2cffefe749e916b6527acd6 \ + --hash=sha256:ca0746a19ec421219f4d713f848fa297a661a8a8c1504867e55bfb5e09091509 # via # gcp-releasetool # twine -markupsafe==2.1.1 \ - --hash=sha256:0212a68688482dc52b2d45013df70d169f542b7394fc744c02a57374a4207003 \ - --hash=sha256:089cf3dbf0cd6c100f02945abeb18484bd1ee57a079aefd52cffd17fba910b88 \ - --hash=sha256:10c1bfff05d95783da83491be968e8fe789263689c02724e0c691933c52994f5 \ - --hash=sha256:33b74d289bd2f5e527beadcaa3f401e0df0a89927c1559c8566c066fa4248ab7 \ - --hash=sha256:3799351e2336dc91ea70b034983ee71cf2f9533cdff7c14c90ea126bfd95d65a \ - --hash=sha256:3ce11ee3f23f79dbd06fb3d63e2f6af7b12db1d46932fe7bd8afa259a5996603 \ - --hash=sha256:421be9fbf0ffe9ffd7a378aafebbf6f4602d564d34be190fc19a193232fd12b1 \ - --hash=sha256:43093fb83d8343aac0b1baa75516da6092f58f41200907ef92448ecab8825135 \ - --hash=sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247 \ - --hash=sha256:4a33dea2b688b3190ee12bd7cfa29d39c9ed176bda40bfa11099a3ce5d3a7ac6 \ - --hash=sha256:4b9fe39a2ccc108a4accc2676e77da025ce383c108593d65cc909add5c3bd601 \ - --hash=sha256:56442863ed2b06d19c37f94d999035e15ee982988920e12a5b4ba29b62ad1f77 \ - --hash=sha256:671cd1187ed5e62818414afe79ed29da836dde67166a9fac6d435873c44fdd02 \ - --hash=sha256:694deca8d702d5db21ec83983ce0bb4b26a578e71fbdbd4fdcd387daa90e4d5e \ - --hash=sha256:6a074d34ee7a5ce3effbc526b7083ec9731bb3cbf921bbe1d3005d4d2bdb3a63 \ - --hash=sha256:6d0072fea50feec76a4c418096652f2c3238eaa014b2f94aeb1d56a66b41403f \ - --hash=sha256:6fbf47b5d3728c6aea2abb0589b5d30459e369baa772e0f37a0320185e87c980 \ - --hash=sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b \ - --hash=sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812 \ - --hash=sha256:8dc1c72a69aa7e082593c4a203dcf94ddb74bb5c8a731e4e1eb68d031e8498ff \ - --hash=sha256:8e3dcf21f367459434c18e71b2a9532d96547aef8a871872a5bd69a715c15f96 \ - --hash=sha256:8e576a51ad59e4bfaac456023a78f6b5e6e7651dcd383bcc3e18d06f9b55d6d1 \ - --hash=sha256:96e37a3dc86e80bf81758c152fe66dbf60ed5eca3d26305edf01892257049925 \ - --hash=sha256:97a68e6ada378df82bc9f16b800ab77cbf4b2fada0081794318520138c088e4a \ - --hash=sha256:99a2a507ed3ac881b975a2976d59f38c19386d128e7a9a18b7df6fff1fd4c1d6 \ - --hash=sha256:a49907dd8420c5685cfa064a1335b6754b74541bbb3706c259c02ed65b644b3e \ - --hash=sha256:b09bf97215625a311f669476f44b8b318b075847b49316d3e28c08e41a7a573f \ - --hash=sha256:b7bd98b796e2b6553da7225aeb61f447f80a1ca64f41d83612e6139ca5213aa4 \ - --hash=sha256:b87db4360013327109564f0e591bd2a3b318547bcef31b468a92ee504d07ae4f \ - --hash=sha256:bcb3ed405ed3222f9904899563d6fc492ff75cce56cba05e32eff40e6acbeaa3 \ - --hash=sha256:d4306c36ca495956b6d568d276ac11fdd9c30a36f1b6eb928070dc5360b22e1c \ - --hash=sha256:d5ee4f386140395a2c818d149221149c54849dfcfcb9f1debfe07a8b8bd63f9a \ - --hash=sha256:dda30ba7e87fbbb7eab1ec9f58678558fd9a6b8b853530e176eabd064da81417 \ - --hash=sha256:e04e26803c9c3851c931eac40c695602c6295b8d432cbe78609649ad9bd2da8a \ - --hash=sha256:e1c0b87e09fa55a220f058d1d49d3fb8df88fbfab58558f1198e08c1e1de842a \ - --hash=sha256:e72591e9ecd94d7feb70c1cbd7be7b3ebea3f548870aa91e2732960fa4d57a37 \ - --hash=sha256:e8c843bbcda3a2f1e3c2ab25913c80a3c5376cd00c6e8c4a86a89a28c8dc5452 \ - --hash=sha256:efc1913fd2ca4f334418481c7e595c00aad186563bbc1ec76067848c7ca0a933 \ - --hash=sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a \ - --hash=sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7 +markdown-it-py==3.0.0 \ + --hash=sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1 \ + --hash=sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb + # via rich +markupsafe==2.1.3 \ + --hash=sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e \ + --hash=sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e \ + --hash=sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431 \ + --hash=sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686 \ + --hash=sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c \ + --hash=sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559 \ + --hash=sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc \ + --hash=sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb \ + --hash=sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939 \ + --hash=sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c \ + --hash=sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0 \ + --hash=sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4 \ + --hash=sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9 \ + --hash=sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575 \ + --hash=sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba \ + --hash=sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d \ + --hash=sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd \ + --hash=sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3 \ + --hash=sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00 \ + --hash=sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155 \ + --hash=sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac \ + --hash=sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52 \ + --hash=sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f \ + --hash=sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8 \ + --hash=sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b \ + --hash=sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007 \ + --hash=sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24 \ + --hash=sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea \ + --hash=sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198 \ + --hash=sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0 \ + --hash=sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee \ + --hash=sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be \ + --hash=sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2 \ + --hash=sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1 \ + --hash=sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707 \ + --hash=sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6 \ + --hash=sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c \ + --hash=sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58 \ + --hash=sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823 \ + --hash=sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779 \ + --hash=sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636 \ + --hash=sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c \ + --hash=sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad \ + --hash=sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee \ + --hash=sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc \ + --hash=sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2 \ + --hash=sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48 \ + --hash=sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7 \ + --hash=sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e \ + --hash=sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b \ + --hash=sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa \ + --hash=sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5 \ + --hash=sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e \ + --hash=sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb \ + --hash=sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9 \ + --hash=sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57 \ + --hash=sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc \ + --hash=sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc \ + --hash=sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2 \ + --hash=sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11 # via jinja2 -more-itertools==9.0.0 \ - --hash=sha256:250e83d7e81d0c87ca6bd942e6aeab8cc9daa6096d12c5308f3f92fa5e5c1f41 \ - --hash=sha256:5a6257e40878ef0520b1803990e3e22303a41b5714006c32a3fd8304b26ea1ab +mdurl==0.1.2 \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ + --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba + # via markdown-it-py +more-itertools==10.1.0 \ + --hash=sha256:626c369fa0eb37bac0291bce8259b332fd59ac792fa5497b59837309cd5b114a \ + --hash=sha256:64e0735fcfdc6f3464ea133afe8ea4483b1c5fe3a3d69852e6503b43a0b222e6 # via jaraco-classes -nox==2022.11.21 \ - --hash=sha256:0e41a990e290e274cb205a976c4c97ee3c5234441a8132c8c3fd9ea3c22149eb \ - --hash=sha256:e21c31de0711d1274ca585a2c5fde36b1aa962005ba8e9322bf5eeed16dcd684 +nh3==0.2.14 \ + --hash=sha256:116c9515937f94f0057ef50ebcbcc10600860065953ba56f14473ff706371873 \ + --hash=sha256:18415df36db9b001f71a42a3a5395db79cf23d556996090d293764436e98e8ad \ + --hash=sha256:203cac86e313cf6486704d0ec620a992c8bc164c86d3a4fd3d761dd552d839b5 \ + --hash=sha256:2b0be5c792bd43d0abef8ca39dd8acb3c0611052ce466d0401d51ea0d9aa7525 \ + --hash=sha256:377aaf6a9e7c63962f367158d808c6a1344e2b4f83d071c43fbd631b75c4f0b2 \ + --hash=sha256:525846c56c2bcd376f5eaee76063ebf33cf1e620c1498b2a40107f60cfc6054e \ + --hash=sha256:5529a3bf99402c34056576d80ae5547123f1078da76aa99e8ed79e44fa67282d \ + --hash=sha256:7771d43222b639a4cd9e341f870cee336b9d886de1ad9bec8dddab22fe1de450 \ + --hash=sha256:88c753efbcdfc2644a5012938c6b9753f1c64a5723a67f0301ca43e7b85dcf0e \ + --hash=sha256:93a943cfd3e33bd03f77b97baa11990148687877b74193bf777956b67054dcc6 \ + --hash=sha256:9be2f68fb9a40d8440cbf34cbf40758aa7f6093160bfc7fb018cce8e424f0c3a \ + --hash=sha256:a0c509894fd4dccdff557068e5074999ae3b75f4c5a2d6fb5415e782e25679c4 \ + --hash=sha256:ac8056e937f264995a82bf0053ca898a1cb1c9efc7cd68fa07fe0060734df7e4 \ + --hash=sha256:aed56a86daa43966dd790ba86d4b810b219f75b4bb737461b6886ce2bde38fd6 \ + --hash=sha256:e8986f1dd3221d1e741fda0a12eaa4a273f1d80a35e31a1ffe579e7c621d069e \ + --hash=sha256:f99212a81c62b5f22f9e7c3e347aa00491114a5647e1f13bbebd79c3e5f08d75 + # via readme-renderer +nox==2023.4.22 \ + --hash=sha256:0b1adc619c58ab4fa57d6ab2e7823fe47a32e70202f287d78474adcc7bda1891 \ + --hash=sha256:46c0560b0dc609d7d967dc99e22cb463d3c4caf54a5fda735d6c11b5177e3a9f # via -r requirements.in -packaging==21.3 \ - --hash=sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb \ - --hash=sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522 +packaging==23.2 \ + --hash=sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5 \ + --hash=sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7 # via # gcp-releasetool # nox -pkginfo==1.8.3 \ - --hash=sha256:848865108ec99d4901b2f7e84058b6e7660aae8ae10164e015a6dcf5b242a594 \ - --hash=sha256:a84da4318dd86f870a9447a8c98340aa06216bfc6f2b7bdc4b8766984ae1867c +pkginfo==1.9.6 \ + --hash=sha256:4b7a555a6d5a22169fcc9cf7bfd78d296b0361adad412a346c1226849af5e546 \ + --hash=sha256:8fd5896e8718a4372f0ea9cc9d96f6417c9b986e23a4d116dda26b62cc29d046 # via twine -platformdirs==2.5.4 \ - --hash=sha256:1006647646d80f16130f052404c6b901e80ee4ed6bef6792e1f238a8969106f7 \ - --hash=sha256:af0276409f9a02373d540bf8480021a048711d572745aef4b7842dad245eba10 +platformdirs==3.11.0 \ + --hash=sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3 \ + --hash=sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e # via virtualenv protobuf==3.20.3 \ --hash=sha256:03038ac1cfbc41aa21f6afcbcd357281d7521b4157926f30ebecc8d4ea59dcb7 \ @@ -383,34 +411,30 @@ protobuf==3.20.3 \ # gcp-releasetool # google-api-core # googleapis-common-protos -pyasn1==0.4.8 \ - --hash=sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d \ - --hash=sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba +pyasn1==0.5.0 \ + --hash=sha256:87a2121042a1ac9358cabcaf1d07680ff97ee6404333bacca15f76aa8ad01a57 \ + --hash=sha256:97b7290ca68e62a832558ec3976f15cbf911bf5d7c7039d8b861c2a0ece69fde # via # pyasn1-modules # rsa -pyasn1-modules==0.2.8 \ - --hash=sha256:905f84c712230b2c592c19470d3ca8d552de726050d1d1716282a1f6146be65e \ - --hash=sha256:a50b808ffeb97cb3601dd25981f6b016cbb3d31fbf57a8b8a87428e6158d0c74 +pyasn1-modules==0.3.0 \ + --hash=sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c \ + --hash=sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d # via google-auth pycparser==2.21 \ --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206 # via cffi -pygments==2.15.0 \ - --hash=sha256:77a3299119af881904cd5ecd1ac6a66214b6e9bed1f2db16993b54adede64094 \ - --hash=sha256:f7e36cffc4c517fbc252861b9a6e4644ca0e5abadf9a113c72d1358ad09b9500 +pygments==2.16.1 \ + --hash=sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692 \ + --hash=sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29 # via # readme-renderer # rich -pyjwt==2.6.0 \ - --hash=sha256:69285c7e31fc44f68a1feb309e948e0df53259d579295e6cfe2b1792329f05fd \ - --hash=sha256:d83c3d892a77bbb74d3e1a2cfa90afaadb60945205d1095d9221f04466f64c14 +pyjwt==2.8.0 \ + --hash=sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de \ + --hash=sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320 # via gcp-releasetool -pyparsing==3.0.9 \ - --hash=sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb \ - --hash=sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc - # via packaging pyperclip==1.8.2 \ --hash=sha256:105254a8b04934f0bc84e9c24eb360a591aaf6535c9def5f29d92af107a9bf57 # via gcp-releasetool @@ -418,9 +442,9 @@ python-dateutil==2.8.2 \ --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 # via gcp-releasetool -readme-renderer==37.3 \ - --hash=sha256:cd653186dfc73055656f090f227f5cb22a046d7f71a841dfa305f55c9a513273 \ - --hash=sha256:f67a16caedfa71eef48a31b39708637a6f4664c4394801a7b0d6432d13907343 +readme-renderer==42.0 \ + --hash=sha256:13d039515c1f24de668e2c93f2e877b9dbe6c6c32328b90a40a49d8b2b85f36d \ + --hash=sha256:2d55489f83be4992fe4454939d1a051c33edbab778e82761d060c9fc6b308cd1 # via twine requests==2.31.0 \ --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \ @@ -431,17 +455,17 @@ requests==2.31.0 \ # google-cloud-storage # requests-toolbelt # twine -requests-toolbelt==0.10.1 \ - --hash=sha256:18565aa58116d9951ac39baa288d3adb5b3ff975c4f25eee78555d89e8f247f7 \ - --hash=sha256:62e09f7ff5ccbda92772a29f394a49c3ad6cb181d568b1337626b2abb628a63d +requests-toolbelt==1.0.0 \ + --hash=sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6 \ + --hash=sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06 # via twine rfc3986==2.0.0 \ --hash=sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd \ --hash=sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c # via twine -rich==12.6.0 \ - --hash=sha256:a4eb26484f2c82589bd9a17c73d32a010b1e29d89f1604cd9bf3a2097b81bb5e \ - --hash=sha256:ba3a3775974105c221d31141f2c116f4fd65c5ceb0698657a11e9f295ec93fd0 +rich==13.6.0 \ + --hash=sha256:2b38e2fe9ca72c9a00170a1a2d20c63c790d0e10ef1fe35eba76e1e7b1d7d245 \ + --hash=sha256:5c14d22737e6d5084ef4771b62d5d4363165b403455a30a1c8ca39dc7b644bef # via twine rsa==4.9 \ --hash=sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7 \ @@ -455,43 +479,37 @@ six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 # via - # bleach # gcp-docuploader - # google-auth # python-dateutil -twine==4.0.1 \ - --hash=sha256:42026c18e394eac3e06693ee52010baa5313e4811d5a11050e7d48436cf41b9e \ - --hash=sha256:96b1cf12f7ae611a4a40b6ae8e9570215daff0611828f5fe1f37a16255ab24a0 +twine==4.0.2 \ + --hash=sha256:929bc3c280033347a00f847236564d1c52a3e61b1ac2516c97c48f3ceab756d8 \ + --hash=sha256:9e102ef5fdd5a20661eb88fad46338806c3bd32cf1db729603fe3697b1bc83c8 # via -r requirements.in -typing-extensions==4.4.0 \ - --hash=sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa \ - --hash=sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e +typing-extensions==4.8.0 \ + --hash=sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0 \ + --hash=sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef # via -r requirements.in -urllib3==1.26.18 \ - --hash=sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07 \ - --hash=sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0 +urllib3==2.0.7 \ + --hash=sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84 \ + --hash=sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e # via # requests # twine -virtualenv==20.16.7 \ - --hash=sha256:8691e3ff9387f743e00f6bb20f70121f5e4f596cae754531f2b3b3a1b1ac696e \ - --hash=sha256:efd66b00386fdb7dbe4822d172303f40cd05e50e01740b19ea42425cbe653e29 +virtualenv==20.24.6 \ + --hash=sha256:02ece4f56fbf939dbbc33c0715159951d6bf14aaf5457b092e4548e1382455af \ + --hash=sha256:520d056652454c5098a00c0f073611ccbea4c79089331f60bf9d7ba247bb7381 # via nox -webencodings==0.5.1 \ - --hash=sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78 \ - --hash=sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923 - # via bleach -wheel==0.38.4 \ - --hash=sha256:965f5259b566725405b05e7cf774052044b1ed30119b5d586b2703aafe8719ac \ - --hash=sha256:b60533f3f5d530e971d6737ca6d58681ee434818fab630c83a734bb10c083ce8 +wheel==0.41.3 \ + --hash=sha256:488609bc63a29322326e05560731bf7bfea8e48ad646e1f5e40d366607de0942 \ + --hash=sha256:4d4987ce51a49370ea65c0bfd2234e8ce80a12780820d9dc462597a6e60d0841 # via -r requirements.in -zipp==3.10.0 \ - --hash=sha256:4fcb6f278987a6605757302a6e40e896257570d11c51628968ccb2a47e80c6c1 \ - --hash=sha256:7a7262fd930bd3e36c50b9a64897aec3fafff3dfdeec9623ae22b40e93f99bb8 +zipp==3.17.0 \ + --hash=sha256:0e923e726174922dce09c53c59ad483ff7bbb8e572e00c7f7c46b88556409f31 \ + --hash=sha256:84e64a1c28cf7e91ed2078bb8cc8c259cb19b76942096c8d7b84947690cabaf0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -setuptools==65.5.1 \ - --hash=sha256:d0b9a8433464d5800cbe05094acf5c6d52a91bfac9b52bcfc4d41382be5d5d31 \ - --hash=sha256:e197a19aa8ec9722928f2206f8de752def0e4c9fc6953527360d1c36d94ddb2f +setuptools==68.2.2 \ + --hash=sha256:4ac1475276d2f1c48684874089fefcd83bd7162ddaafb81fac866ba0db282a87 \ + --hash=sha256:b454a35605876da60632df1a60f736524eb73cc47bbc9f3f1ef1b644de74fd2a # via -r requirements.in From 61277946c9f9f99385a3431422cb1097f33f3de0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Nov 2023 16:38:59 -0800 Subject: [PATCH 370/519] chore(deps): bump pyarrow from 12.0.1 to 14.0.1 in /samples/snippets (#696) * chore(deps): bump pyarrow from 12.0.1 to 14.0.1 in /samples/snippets Bumps [pyarrow](https://github.com/apache/arrow) from 12.0.1 to 14.0.1. - [Commits](https://github.com/apache/arrow/compare/go/v12.0.1...apache-arrow-14.0.1) --- updated-dependencies: - dependency-name: pyarrow dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Update requirements.txt --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Lingqing Gan --- packages/pandas-gbq/samples/snippets/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 3aace91bfa7f..3876d6601117 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -3,4 +3,5 @@ google-cloud-bigquery==3.11.4 pandas-gbq==0.19.2 pandas===1.3.5; python_version == '3.7' pandas==2.0.3; python_version >= '3.8' -pyarrow==12.0.1 +pyarrow==12.0.1; python_version == '3.7' +pyarrow==14.0.1; python_version >= '3.8' From 4982ea92644263eba0833e71f71968395e981d00 Mon Sep 17 00:00:00 2001 From: Lingqing Gan Date: Fri, 17 Nov 2023 15:00:35 -0500 Subject: [PATCH 371/519] chore: include api error info in raised errors (#699) * chore: include api error info in raised errors * format strings --- packages/pandas-gbq/pandas_gbq/gbq.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 26a566d9c414..3d43884a81f5 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -483,13 +483,15 @@ def run_query(self, query, max_results=None, progress_bar_type=None, **kwargs): project=self.project_id, ) logger.debug("Query running...") - except (RefreshError, ValueError): + except (RefreshError, ValueError) as ex: if self.private_key: - raise AccessDenied("The service account credentials are not valid") + raise AccessDenied( + f"The service account credentials are not valid: {ex}" + ) else: raise AccessDenied( "The credentials have been revoked or expired, " - "please re-run the application to re-authorize" + f"please re-run the application to re-authorize: {ex}" ) except self.http_error as ex: self.process_http_error(ex) From 9a9bd6b8be7bb6cd105afc915292e2526d3ce22a Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 1 Dec 2023 10:56:53 -0800 Subject: [PATCH 372/519] chore: bump cryptography from 41.0.5 to 41.0.6 in /synthtool/gcp/templates/python_library/.kokoro (#704) Source-Link: https://github.com/googleapis/synthtool/commit/9367caadcbb30b5b2719f30eb00c44cc913550ed Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:2f155882785883336b4468d5218db737bb1d10c9cea7cb62219ad16fe248c03c Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 +- packages/pandas-gbq/.kokoro/requirements.txt | 48 +++++++++---------- .../.kokoro/samples/python3.12/common.cfg | 40 ++++++++++++++++ .../.kokoro/samples/python3.12/continuous.cfg | 6 +++ .../samples/python3.12/periodic-head.cfg | 11 +++++ .../.kokoro/samples/python3.12/periodic.cfg | 6 +++ .../.kokoro/samples/python3.12/presubmit.cfg | 6 +++ .../pandas-gbq/samples/snippets/noxfile.py | 2 +- 8 files changed, 96 insertions(+), 27 deletions(-) create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.12/common.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.12/continuous.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.12/periodic-head.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.12/periodic.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.12/presubmit.cfg diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 453b540c1e58..773c1dfd2146 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:caffe0a9277daeccc4d1de5c9b55ebba0901b57c2f713ec9c876b0d4ec064f61 -# created: 2023-11-08T19:46:45.022803742Z + digest: sha256:2f155882785883336b4468d5218db737bb1d10c9cea7cb62219ad16fe248c03c +# created: 2023-11-29T14:54:29.548172703Z diff --git a/packages/pandas-gbq/.kokoro/requirements.txt b/packages/pandas-gbq/.kokoro/requirements.txt index 8957e21104e2..e5c1ffca94b7 100644 --- a/packages/pandas-gbq/.kokoro/requirements.txt +++ b/packages/pandas-gbq/.kokoro/requirements.txt @@ -93,30 +93,30 @@ colorlog==6.7.0 \ # via # gcp-docuploader # nox -cryptography==41.0.5 \ - --hash=sha256:0c327cac00f082013c7c9fb6c46b7cc9fa3c288ca702c74773968173bda421bf \ - --hash=sha256:0d2a6a598847c46e3e321a7aef8af1436f11c27f1254933746304ff014664d84 \ - --hash=sha256:227ec057cd32a41c6651701abc0328135e472ed450f47c2766f23267b792a88e \ - --hash=sha256:22892cc830d8b2c89ea60148227631bb96a7da0c1b722f2aac8824b1b7c0b6b8 \ - --hash=sha256:392cb88b597247177172e02da6b7a63deeff1937fa6fec3bbf902ebd75d97ec7 \ - --hash=sha256:3be3ca726e1572517d2bef99a818378bbcf7d7799d5372a46c79c29eb8d166c1 \ - --hash=sha256:573eb7128cbca75f9157dcde974781209463ce56b5804983e11a1c462f0f4e88 \ - --hash=sha256:580afc7b7216deeb87a098ef0674d6ee34ab55993140838b14c9b83312b37b86 \ - --hash=sha256:5a70187954ba7292c7876734183e810b728b4f3965fbe571421cb2434d279179 \ - --hash=sha256:73801ac9736741f220e20435f84ecec75ed70eda90f781a148f1bad546963d81 \ - --hash=sha256:7d208c21e47940369accfc9e85f0de7693d9a5d843c2509b3846b2db170dfd20 \ - --hash=sha256:8254962e6ba1f4d2090c44daf50a547cd5f0bf446dc658a8e5f8156cae0d8548 \ - --hash=sha256:88417bff20162f635f24f849ab182b092697922088b477a7abd6664ddd82291d \ - --hash=sha256:a48e74dad1fb349f3dc1d449ed88e0017d792997a7ad2ec9587ed17405667e6d \ - --hash=sha256:b948e09fe5fb18517d99994184854ebd50b57248736fd4c720ad540560174ec5 \ - --hash=sha256:c707f7afd813478e2019ae32a7c49cd932dd60ab2d2a93e796f68236b7e1fbf1 \ - --hash=sha256:d38e6031e113b7421db1de0c1b1f7739564a88f1684c6b89234fbf6c11b75147 \ - --hash=sha256:d3977f0e276f6f5bf245c403156673db103283266601405376f075c849a0b936 \ - --hash=sha256:da6a0ff8f1016ccc7477e6339e1d50ce5f59b88905585f77193ebd5068f1e797 \ - --hash=sha256:e270c04f4d9b5671ebcc792b3ba5d4488bf7c42c3c241a3748e2599776f29696 \ - --hash=sha256:e886098619d3815e0ad5790c973afeee2c0e6e04b4da90b88e6bd06e2a0b1b72 \ - --hash=sha256:ec3b055ff8f1dce8e6ef28f626e0972981475173d7973d63f271b29c8a2897da \ - --hash=sha256:fba1e91467c65fe64a82c689dc6cf58151158993b13eb7a7f3f4b7f395636723 +cryptography==41.0.6 \ + --hash=sha256:068bc551698c234742c40049e46840843f3d98ad7ce265fd2bd4ec0d11306596 \ + --hash=sha256:0f27acb55a4e77b9be8d550d762b0513ef3fc658cd3eb15110ebbcbd626db12c \ + --hash=sha256:2132d5865eea673fe6712c2ed5fb4fa49dba10768bb4cc798345748380ee3660 \ + --hash=sha256:3288acccef021e3c3c10d58933f44e8602cf04dba96d9796d70d537bb2f4bbc4 \ + --hash=sha256:35f3f288e83c3f6f10752467c48919a7a94b7d88cc00b0668372a0d2ad4f8ead \ + --hash=sha256:398ae1fc711b5eb78e977daa3cbf47cec20f2c08c5da129b7a296055fbb22aed \ + --hash=sha256:422e3e31d63743855e43e5a6fcc8b4acab860f560f9321b0ee6269cc7ed70cc3 \ + --hash=sha256:48783b7e2bef51224020efb61b42704207dde583d7e371ef8fc2a5fb6c0aabc7 \ + --hash=sha256:4d03186af98b1c01a4eda396b137f29e4e3fb0173e30f885e27acec8823c1b09 \ + --hash=sha256:5daeb18e7886a358064a68dbcaf441c036cbdb7da52ae744e7b9207b04d3908c \ + --hash=sha256:60e746b11b937911dc70d164060d28d273e31853bb359e2b2033c9e93e6f3c43 \ + --hash=sha256:742ae5e9a2310e9dade7932f9576606836ed174da3c7d26bc3d3ab4bd49b9f65 \ + --hash=sha256:7e00fb556bda398b99b0da289ce7053639d33b572847181d6483ad89835115f6 \ + --hash=sha256:85abd057699b98fce40b41737afb234fef05c67e116f6f3650782c10862c43da \ + --hash=sha256:8efb2af8d4ba9dbc9c9dd8f04d19a7abb5b49eab1f3694e7b5a16a5fc2856f5c \ + --hash=sha256:ae236bb8760c1e55b7a39b6d4d32d2279bc6c7c8500b7d5a13b6fb9fc97be35b \ + --hash=sha256:afda76d84b053923c27ede5edc1ed7d53e3c9f475ebaf63c68e69f1403c405a8 \ + --hash=sha256:b27a7fd4229abef715e064269d98a7e2909ebf92eb6912a9603c7e14c181928c \ + --hash=sha256:b648fe2a45e426aaee684ddca2632f62ec4613ef362f4d681a9a6283d10e079d \ + --hash=sha256:c5a550dc7a3b50b116323e3d376241829fd326ac47bc195e04eb33a8170902a9 \ + --hash=sha256:da46e2b5df770070412c46f87bac0849b8d685c5f2679771de277a422c7d0b86 \ + --hash=sha256:f39812f70fc5c71a15aa3c97b2bbe213c3f2a460b79bd21c40d033bb34a9bf36 \ + --hash=sha256:ff369dd19e8fe0528b02e8df9f2aeb2479f89b1270d90f96a63500afe9af5cae # via # gcp-releasetool # secretstorage diff --git a/packages/pandas-gbq/.kokoro/samples/python3.12/common.cfg b/packages/pandas-gbq/.kokoro/samples/python3.12/common.cfg new file mode 100644 index 000000000000..17c144c4e601 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.12/common.cfg @@ -0,0 +1,40 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.12" +} + +# Declare build specific Cloud project. +env_vars: { + key: "BUILD_SPECIFIC_GCLOUD_PROJECT" + value: "python-docs-samples-tests-312" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-pandas/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-bigquery-pandas/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.12/continuous.cfg b/packages/pandas-gbq/.kokoro/samples/python3.12/continuous.cfg new file mode 100644 index 000000000000..a1c8d9759c88 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.12/continuous.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.12/periodic-head.cfg b/packages/pandas-gbq/.kokoro/samples/python3.12/periodic-head.cfg new file mode 100644 index 000000000000..98efde4dc99d --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.12/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-pandas/.kokoro/test-samples-against-head.sh" +} diff --git a/packages/pandas-gbq/.kokoro/samples/python3.12/periodic.cfg b/packages/pandas-gbq/.kokoro/samples/python3.12/periodic.cfg new file mode 100644 index 000000000000..71cd1e597e38 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.12/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} diff --git a/packages/pandas-gbq/.kokoro/samples/python3.12/presubmit.cfg b/packages/pandas-gbq/.kokoro/samples/python3.12/presubmit.cfg new file mode 100644 index 000000000000..a1c8d9759c88 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.12/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/packages/pandas-gbq/samples/snippets/noxfile.py b/packages/pandas-gbq/samples/snippets/noxfile.py index 1224cbe212e4..3b7135946fd5 100644 --- a/packages/pandas-gbq/samples/snippets/noxfile.py +++ b/packages/pandas-gbq/samples/snippets/noxfile.py @@ -89,7 +89,7 @@ def get_pytest_env_vars() -> Dict[str, str]: # DO NOT EDIT - automatically generated. # All versions used to test samples. -ALL_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11"] +ALL_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] # Any default versions that should be ignored. IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] From 6d41a15d22eeedc3b613a75ed0d87618f1481953 Mon Sep 17 00:00:00 2001 From: Kira Date: Wed, 6 Dec 2023 09:33:56 -0800 Subject: [PATCH 373/519] feat: add 'columns' as an alias for 'col_order' (#701) * feat: add 'columns' as an alias for 'col_order' * Added test to test alias correctness * reformatted with black * refactored to alias checking and testing * Reformatted tests for columns alias * feat: add 'columns' as an alias for 'col_order' * Added test to test alias correctness * reformatted with black * refactored to alias checking and testing * Reformatted tests for columns alias * Made col_order a keyword argument and added to-do * Edit todo comment * Fixed small error in docstring * Fixed valueerror message * reformatted with black --------- Co-authored-by: Chalmer Lowe --- packages/pandas-gbq/docs/reading.rst | 2 +- packages/pandas-gbq/pandas_gbq/gbq.py | 23 ++++++++++--- packages/pandas-gbq/tests/system/test_gbq.py | 34 ++++++++++++++++++++ 3 files changed, 53 insertions(+), 6 deletions(-) diff --git a/packages/pandas-gbq/docs/reading.rst b/packages/pandas-gbq/docs/reading.rst index c5e814bfdce7..6361280ac5da 100644 --- a/packages/pandas-gbq/docs/reading.rst +++ b/packages/pandas-gbq/docs/reading.rst @@ -28,7 +28,7 @@ destination DataFrame as well as a preferred column order as follows: 'SELECT * FROM `test_dataset.test_table`', project_id=projectid, index_col='index_column_name', - col_order=['col1', 'col2', 'col3']) + columns=['col1', 'col2']) Querying with legacy SQL syntax ------------------------------- diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 3d43884a81f5..d4a8d2b7751b 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -734,7 +734,7 @@ def read_gbq( query_or_table, project_id=None, index_col=None, - col_order=None, + columns=None, reauth=False, auth_local_webserver=True, dialect=None, @@ -750,6 +750,8 @@ def read_gbq( auth_redirect_uri=None, client_id=None, client_secret=None, + *, + col_order=None, ): r"""Load data from Google BigQuery using google-cloud-python @@ -773,7 +775,7 @@ def read_gbq( the environment. index_col : str, optional Name of result column to use for index in results DataFrame. - col_order : list(str), optional + columns : list(str), optional List of BigQuery column names in the desired order for results DataFrame. reauth : boolean, default False @@ -888,6 +890,8 @@ def read_gbq( client_secret : str The Client Secret associated with the Client ID for the Google Cloud Project the user is attempting to connect to. + col_order : list(str), optional + Alias for columns, retained for backwards compatibility. Returns ------- @@ -966,10 +970,19 @@ def read_gbq( 'Index column "{0}" does not exist in DataFrame.'.format(index_col) ) + # Using columns as an alias for col_order, raising an error if both provided + if col_order and not columns: + columns = col_order + elif col_order and columns: + raise ValueError( + "Must specify either columns (preferred) or col_order, not both" + ) + # Change the order of columns in the DataFrame based on provided list - if col_order is not None: - if sorted(col_order) == sorted(final_df.columns): - final_df = final_df[col_order] + # TODO(kiraksi): allow columns to be a subset of all columns in the table, with follow up PR + if columns is not None: + if sorted(columns) == sorted(final_df.columns): + final_df = final_df[columns] else: raise InvalidColumnOrder("Column order does not match this DataFrame.") diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index 9aac2357e651..bc0782645e7f 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -600,6 +600,40 @@ def test_tokyo(self, tokyo_dataset, tokyo_table, project_id): ) assert df["max_year"][0] >= 2000 + def test_columns_as_alias(self, project_id): + query = "SELECT 'a' AS string_1, 'b' AS string_2, 'c' AS string_3" + columns = ["string_2", "string_1", "string_3"] + + df = gbq.read_gbq( + query, + project_id=project_id, + columns=columns, + credentials=self.credentials, + dialect="standard", + ) + + expected = DataFrame({"string_1": ["a"], "string_2": ["b"], "string_3": ["c"]})[ + columns + ] + + # Verify that the result_frame matches the expected DataFrame + tm.assert_frame_equal(df, expected) + + def test_columns_and_col_order_raises_error(self, project_id): + query = "SELECT 'a' AS string_1, 'b' AS string_2, 'c' AS string_3" + columns = ["string_2", "string_1"] + col_order = ["string_3", "string_1", "string_2"] + + with pytest.raises(ValueError): + gbq.read_gbq( + query, + project_id=project_id, + columns=columns, + col_order=col_order, + credentials=self.credentials, + dialect="standard", + ) + class TestToGBQIntegration(object): @pytest.fixture(autouse=True, scope="function") From 091976616774fd99a2d9d0d99b542728a08ae72a Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Wed, 6 Dec 2023 14:28:02 -0800 Subject: [PATCH 374/519] feat: Add support for Python 3.12 (#702) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(python): Add Python 3.12 Source-Link: https://github.com/googleapis/synthtool/commit/af16e6d4672cc7b400f144de2fc3068b54ff47d2 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:bacc3af03bff793a03add584537b36b5644342931ad989e3ba1171d3bd5399f5 * add python 3.12 to owlbot.py and setup.py * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * Add python 3.12 to noxfile.py * add constraints file for python 3.12 * (test) add module six to fix conda test failure * undo adding six to requirements.txt --------- Co-authored-by: Owl Bot Co-authored-by: Anthonios Partheniou Co-authored-by: Lingqing Gan --- packages/pandas-gbq/.github/workflows/unittest.yml | 2 +- packages/pandas-gbq/CONTRIBUTING.rst | 10 ++++++---- packages/pandas-gbq/noxfile.py | 4 ++-- packages/pandas-gbq/owlbot.py | 4 ++-- packages/pandas-gbq/setup.py | 1 + packages/pandas-gbq/testing/constraints-3.12.txt | 0 6 files changed, 12 insertions(+), 9 deletions(-) create mode 100644 packages/pandas-gbq/testing/constraints-3.12.txt diff --git a/packages/pandas-gbq/.github/workflows/unittest.yml b/packages/pandas-gbq/.github/workflows/unittest.yml index 0de0f6918c27..17369fa2961d 100644 --- a/packages/pandas-gbq/.github/workflows/unittest.yml +++ b/packages/pandas-gbq/.github/workflows/unittest.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] steps: - name: Checkout uses: actions/checkout@v3 diff --git a/packages/pandas-gbq/CONTRIBUTING.rst b/packages/pandas-gbq/CONTRIBUTING.rst index b93f22405720..487deabb5afa 100644 --- a/packages/pandas-gbq/CONTRIBUTING.rst +++ b/packages/pandas-gbq/CONTRIBUTING.rst @@ -22,7 +22,7 @@ In order to add a feature: documentation. - The feature must work fully on the following CPython versions: - 3.7, 3.8, 3.9, 3.10 and 3.11 on both UNIX and Windows. + 3.7, 3.8, 3.9, 3.10, 3.11 and 3.12 on both UNIX and Windows. - The feature must not add unnecessary dependencies (where "unnecessary" is of course subjective, but new dependencies should @@ -72,7 +72,7 @@ We use `nox `__ to instrument our tests. - To run a single unit test:: - $ nox -s unit-3.11 -- -k + $ nox -s unit-3.12 -- -k .. note:: @@ -143,12 +143,12 @@ Running System Tests $ nox -s system # Run a single system test - $ nox -s system-3.11 -- -k + $ nox -s system-3.12 -- -k .. note:: - System tests are only configured to run under Python 3.7, 3.8, 3.9, 3.10 and 3.11. + System tests are only configured to run under Python 3.7, 3.8, 3.9, 3.10, 3.11 and 3.12. For expediency, we do not run them in older versions of Python 3. This alone will not run the tests. You'll need to change some local @@ -226,12 +226,14 @@ We support: - `Python 3.9`_ - `Python 3.10`_ - `Python 3.11`_ +- `Python 3.12`_ .. _Python 3.7: https://docs.python.org/3.7/ .. _Python 3.8: https://docs.python.org/3.8/ .. _Python 3.9: https://docs.python.org/3.9/ .. _Python 3.10: https://docs.python.org/3.10/ .. _Python 3.11: https://docs.python.org/3.11/ +.. _Python 3.12: https://docs.python.org/3.12/ Supported versions can be found in our ``noxfile.py`` `config`_. diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 051f5da4dc00..a0d5a94cf997 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -32,7 +32,7 @@ DEFAULT_PYTHON_VERSION = "3.8" -UNIT_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11"] +UNIT_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] UNIT_TEST_STANDARD_DEPENDENCIES = [ "mock", "asyncmock", @@ -57,7 +57,7 @@ UNIT_TEST_PYTHON_VERSIONS[-1], ] -SYSTEM_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11"] +SYSTEM_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] SYSTEM_TEST_STANDARD_DEPENDENCIES = [ "mock", "pytest", diff --git a/packages/pandas-gbq/owlbot.py b/packages/pandas-gbq/owlbot.py index e648334d7996..e6c59be4b041 100644 --- a/packages/pandas-gbq/owlbot.py +++ b/packages/pandas-gbq/owlbot.py @@ -35,8 +35,8 @@ } extras = ["tqdm"] templated_files = common.py_library( - unit_test_python_versions=["3.7", "3.8", "3.9", "3.10", "3.11"], - system_test_python_versions=["3.7", "3.8", "3.9", "3.10", "3.11"], + unit_test_python_versions=["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"], + system_test_python_versions=["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"], cov_level=96, unit_test_external_dependencies=["freezegun"], unit_test_extras=extras, diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index b94a21dc3675..9c71c3c56c07 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -90,6 +90,7 @@ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Operating System :: OS Independent", "Topic :: Internet", "Topic :: Scientific/Engineering", diff --git a/packages/pandas-gbq/testing/constraints-3.12.txt b/packages/pandas-gbq/testing/constraints-3.12.txt new file mode 100644 index 000000000000..e69de29bb2d1 From 894e73b6764917afc0d9da4ec9481e5155a12a0b Mon Sep 17 00:00:00 2001 From: Kira Date: Thu, 7 Dec 2023 07:43:57 -0800 Subject: [PATCH 375/519] feat: removed pkg_resources for native namespace support (#707) * feat: removed pkg_resources for native namespace compatibility * reformatted with black * removed unecessary imports * removed previous changes, used packaging dependeny * Added packaging to test google api imports * changed importlib to packaging in tests * removed small error * add minimum version to packaging dependency * added packaging to 3.7 constraints --------- Co-authored-by: Chalmer Lowe --- packages/pandas-gbq/pandas_gbq/features.py | 38 +++++++++---------- packages/pandas-gbq/pandas_gbq/gbq.py | 4 +- packages/pandas-gbq/setup.py | 3 +- .../pandas-gbq/testing/constraints-3.7.txt | 1 + packages/pandas-gbq/tests/system/test_gbq.py | 8 +--- 5 files changed, 26 insertions(+), 28 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/features.py b/packages/pandas-gbq/pandas_gbq/features.py index ad20c6402ce4..4b70a14adb17 100644 --- a/packages/pandas-gbq/pandas_gbq/features.py +++ b/packages/pandas-gbq/pandas_gbq/features.py @@ -23,15 +23,15 @@ def __init__(self): @property def bigquery_installed_version(self): import google.cloud.bigquery - import pkg_resources + import packaging.version if self._bigquery_installed_version is not None: return self._bigquery_installed_version - self._bigquery_installed_version = pkg_resources.parse_version( + self._bigquery_installed_version = packaging.version.parse( google.cloud.bigquery.__version__ ) - bigquery_minimum_version = pkg_resources.parse_version(BIGQUERY_MINIMUM_VERSION) + bigquery_minimum_version = packaging.version.parse(BIGQUERY_MINIMUM_VERSION) if self._bigquery_installed_version < bigquery_minimum_version: raise ImportError( @@ -45,68 +45,68 @@ def bigquery_installed_version(self): @property def bigquery_has_accurate_timestamp(self): - import pkg_resources + import packaging.version - min_version = pkg_resources.parse_version(BIGQUERY_ACCURATE_TIMESTAMP_VERSION) + min_version = packaging.version.parse(BIGQUERY_ACCURATE_TIMESTAMP_VERSION) return self.bigquery_installed_version >= min_version @property def bigquery_has_bignumeric(self): - import pkg_resources + import packaging.version - min_version = pkg_resources.parse_version(BIGQUERY_SUPPORTS_BIGNUMERIC_VERSION) + min_version = packaging.version.parse(BIGQUERY_SUPPORTS_BIGNUMERIC_VERSION) return self.bigquery_installed_version >= min_version @property def bigquery_has_from_dataframe_with_csv(self): - import pkg_resources + import packaging.version - bigquery_from_dataframe_version = pkg_resources.parse_version( + bigquery_from_dataframe_version = packaging.version.parse( BIGQUERY_FROM_DATAFRAME_CSV_VERSION ) return self.bigquery_installed_version >= bigquery_from_dataframe_version @property def bigquery_needs_date_as_object(self): - import pkg_resources + import packaging.version - max_version = pkg_resources.parse_version(BIGQUERY_NO_DATE_AS_OBJECT_VERSION) + max_version = packaging.version.parse(BIGQUERY_NO_DATE_AS_OBJECT_VERSION) return self.bigquery_installed_version < max_version @property def pandas_installed_version(self): import pandas - import pkg_resources + import packaging.version if self._pandas_installed_version is not None: return self._pandas_installed_version - self._pandas_installed_version = pkg_resources.parse_version(pandas.__version__) + self._pandas_installed_version = packaging.version.parse(pandas.__version__) return self._pandas_installed_version @property def pandas_has_deprecated_verbose(self): - import pkg_resources + import packaging.version # Add check for Pandas version before showing deprecation warning. # https://github.com/pydata/pandas-gbq/issues/157 - pandas_verbosity_deprecation = pkg_resources.parse_version( + pandas_verbosity_deprecation = packaging.version.parse( PANDAS_VERBOSITY_DEPRECATION_VERSION ) return self.pandas_installed_version >= pandas_verbosity_deprecation @property def pandas_has_boolean_dtype(self): - import pkg_resources + import packaging.version - desired_version = pkg_resources.parse_version(PANDAS_BOOLEAN_DTYPE_VERSION) + desired_version = packaging.version.parse(PANDAS_BOOLEAN_DTYPE_VERSION) return self.pandas_installed_version >= desired_version @property def pandas_has_parquet_with_lossless_timestamp(self): - import pkg_resources + import packaging.version - desired_version = pkg_resources.parse_version( + desired_version = packaging.version.parse( PANDAS_PARQUET_LOSSLESS_TIMESTAMP_VERSION ) return self.pandas_installed_version >= desired_version diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index d4a8d2b7751b..dbb9e5b54708 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -35,9 +35,9 @@ def _test_google_api_imports(): try: - import pkg_resources # noqa + import packaging # noqa except ImportError as ex: # pragma: NO COVER - raise ImportError("pandas-gbq requires setuptools") from ex + raise ImportError("pandas-gbq requires db-dtypes") from ex try: import db_dtypes # noqa diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index 9c71c3c56c07..d0b16c2e71bf 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -41,6 +41,7 @@ # indefinitely. https://github.com/pydata/pandas-gbq/issues/343 "google-cloud-bigquery >=3.3.5,<4.0.0dev,!=2.4.*", "google-cloud-bigquery-storage >=2.16.2,<3.0.0dev", + "packaging >=20.0.0", ] extras = { "tqdm": "tqdm>=4.23.0", @@ -63,7 +64,7 @@ # benchmarks, etc. packages = [ package - for package in setuptools.PEP420PackageFinder.find() + for package in setuptools.find_namespace_packages() if package.startswith("pandas_gbq") ] diff --git a/packages/pandas-gbq/testing/constraints-3.7.txt b/packages/pandas-gbq/testing/constraints-3.7.txt index dc26cdee16dc..2a4141bd3c29 100644 --- a/packages/pandas-gbq/testing/constraints-3.7.txt +++ b/packages/pandas-gbq/testing/constraints-3.7.txt @@ -17,3 +17,4 @@ pyarrow==3.0.0 pydata-google-auth==1.5.0 tqdm==4.23.0 protobuf==3.19.5 +packaging==20.0.0 diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index bc0782645e7f..7afa4ae93873 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -14,19 +14,15 @@ import pandas.testing as tm from pandas import DataFrame -try: - import pkg_resources # noqa -except ImportError: - raise ImportError("Could not import pkg_resources (setuptools).") -import pytest import pytz +import pytest from pandas_gbq import gbq import pandas_gbq.schema TABLE_ID = "new_test" -PANDAS_VERSION = pkg_resources.parse_version(pandas.__version__) +PANDAS_VERSION = packaging.version.parse(pandas.__version__) def test_imports(): From ce875e1ec011673a8e7e5e8a881d532245b74a7d Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Sun, 10 Dec 2023 09:01:33 -0500 Subject: [PATCH 376/519] build: update actions/checkout and actions/setup-python (#712) Source-Link: https://github.com/googleapis/synthtool/commit/3551acd1261fd8f616cbfd054cda9bd6d6ac75f4 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:230f7fe8a0d2ed81a519cfc15c6bb11c5b46b9fb449b8b1219b3771bcb520ad2 Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 ++-- packages/pandas-gbq/.github/workflows/docs.yml | 8 ++++---- packages/pandas-gbq/.github/workflows/lint.yml | 4 ++-- packages/pandas-gbq/.github/workflows/unittest.yml | 8 ++++---- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 773c1dfd2146..40bf99731959 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:2f155882785883336b4468d5218db737bb1d10c9cea7cb62219ad16fe248c03c -# created: 2023-11-29T14:54:29.548172703Z + digest: sha256:230f7fe8a0d2ed81a519cfc15c6bb11c5b46b9fb449b8b1219b3771bcb520ad2 +# created: 2023-12-09T15:16:25.430769578Z diff --git a/packages/pandas-gbq/.github/workflows/docs.yml b/packages/pandas-gbq/.github/workflows/docs.yml index 221806cedf58..698fbc5c94da 100644 --- a/packages/pandas-gbq/.github/workflows/docs.yml +++ b/packages/pandas-gbq/.github/workflows/docs.yml @@ -8,9 +8,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.9" - name: Install nox @@ -24,9 +24,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" - name: Install nox diff --git a/packages/pandas-gbq/.github/workflows/lint.yml b/packages/pandas-gbq/.github/workflows/lint.yml index 16d5a9e90f6d..4866193af2a9 100644 --- a/packages/pandas-gbq/.github/workflows/lint.yml +++ b/packages/pandas-gbq/.github/workflows/lint.yml @@ -8,9 +8,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.8" - name: Install nox diff --git a/packages/pandas-gbq/.github/workflows/unittest.yml b/packages/pandas-gbq/.github/workflows/unittest.yml index 17369fa2961d..344ac949aa74 100644 --- a/packages/pandas-gbq/.github/workflows/unittest.yml +++ b/packages/pandas-gbq/.github/workflows/unittest.yml @@ -11,9 +11,9 @@ jobs: python: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - name: Install nox @@ -37,9 +37,9 @@ jobs: - unit steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.8" - name: Install coverage From ae98f6d86a8a5500772dcc8b5fa0f71e9b0f9281 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 12 Dec 2023 09:43:15 -0500 Subject: [PATCH 377/519] chore(main): release 0.20.0 (#650) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- packages/pandas-gbq/CHANGELOG.md | 21 +++++++++++++++++++++ packages/pandas-gbq/pandas_gbq/version.py | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index 505fa802844b..9bbae45eed49 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,26 @@ # Changelog +## [0.20.0](https://github.com/googleapis/python-bigquery-pandas/compare/v0.19.2...v0.20.0) (2023-12-10) + + +### Features + +* Add 'columns' as an alias for 'col_order' ([#701](https://github.com/googleapis/python-bigquery-pandas/issues/701)) ([e52e8f8](https://github.com/googleapis/python-bigquery-pandas/commit/e52e8f85e3f115ee1c9f3dc6733217b04d264f26)) +* Add support for Python 3.12 ([#702](https://github.com/googleapis/python-bigquery-pandas/issues/702)) ([edb93cc](https://github.com/googleapis/python-bigquery-pandas/commit/edb93cc1aee5e7ea931a1d620617be28b4c9a702)) +* Migrating off of circle/ci ([#638](https://github.com/googleapis/python-bigquery-pandas/issues/638)) ([08fe090](https://github.com/googleapis/python-bigquery-pandas/commit/08fe0904375077e43afc03c5529b427cfcf9aa37)) +* Removed pkg_resources for native namespace support ([#707](https://github.com/googleapis/python-bigquery-pandas/issues/707)) ([eeb1959](https://github.com/googleapis/python-bigquery-pandas/commit/eeb19593a75d55e88a2c51ae8eb258c0156d5ba6)) + + +### Bug Fixes + +* Edit units for time in tests ([#655](https://github.com/googleapis/python-bigquery-pandas/issues/655)) ([d4ebb0c](https://github.com/googleapis/python-bigquery-pandas/commit/d4ebb0cb530fde5a28663a4bec8f7f663e2f0c55)) +* Table schema change error ([#692](https://github.com/googleapis/python-bigquery-pandas/issues/692)) ([3dc8ebe](https://github.com/googleapis/python-bigquery-pandas/commit/3dc8ebed4809ad626dcf18124c38930c87750ac8)) + + +### Documentation + +* Migrate .readthedocs.yml to configuration file v2 ([#689](https://github.com/googleapis/python-bigquery-pandas/issues/689)) ([d921219](https://github.com/googleapis/python-bigquery-pandas/commit/d921219690b1b577346f5dc41c0e8d8e5ed2115c)) + ## [0.19.2](https://github.com/googleapis/python-bigquery-pandas/compare/v0.19.1...v0.19.2) (2023-05-10) diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py index ee444a8007b2..ed24fc9ea745 100644 --- a/packages/pandas-gbq/pandas_gbq/version.py +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -2,4 +2,4 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -__version__ = "0.19.2" +__version__ = "0.20.0" From 84ac22c4243816e052c331a8a0fdb9f54081e1fe Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 13 Dec 2023 16:49:27 -0500 Subject: [PATCH 378/519] chore: update required presubmit checks (#713) --- packages/pandas-gbq/.github/sync-repo-settings.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/pandas-gbq/.github/sync-repo-settings.yaml b/packages/pandas-gbq/.github/sync-repo-settings.yaml index 590dda51fa37..a6cf85d3dc36 100644 --- a/packages/pandas-gbq/.github/sync-repo-settings.yaml +++ b/packages/pandas-gbq/.github/sync-repo-settings.yaml @@ -15,6 +15,8 @@ branchProtectionRules: - 'unit (3.8)' - 'unit (3.9)' - 'unit (3.10)' + - 'unit (3.11)' + - 'unit (3.12)' - 'cover' - 'Kokoro' - 'Samples - Lint' @@ -22,6 +24,8 @@ branchProtectionRules: - 'Samples - Python 3.8' - 'Samples - Python 3.9' - 'Samples - Python 3.10' + - 'Samples - Python 3.11' + - 'Samples - Python 3.12' permissionRules: - team: actools-python permission: admin From 870332d7d2a69109de574808b5c60208564bc22c Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Thu, 14 Dec 2023 13:37:24 +0100 Subject: [PATCH 379/519] chore(deps): update all dependencies (#665) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update all dependencies * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot Co-authored-by: Anthonios Partheniou --- packages/pandas-gbq/samples/snippets/requirements-test.txt | 2 +- packages/pandas-gbq/samples/snippets/requirements.txt | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements-test.txt b/packages/pandas-gbq/samples/snippets/requirements-test.txt index 2a4dccc0df2b..00742565d64a 100644 --- a/packages/pandas-gbq/samples/snippets/requirements-test.txt +++ b/packages/pandas-gbq/samples/snippets/requirements-test.txt @@ -1,2 +1,2 @@ google-cloud-testutils==1.3.3 -pytest==7.4.0 +pytest==7.4.1 diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 3876d6601117..d26ba789bd40 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -2,6 +2,7 @@ google-cloud-bigquery-storage==2.22.0 google-cloud-bigquery==3.11.4 pandas-gbq==0.19.2 pandas===1.3.5; python_version == '3.7' -pandas==2.0.3; python_version >= '3.8' +pandas===2.0.3; python_version == '3.8' +pandas==2.1.0; python_version >= '3.9' pyarrow==12.0.1; python_version == '3.7' pyarrow==14.0.1; python_version >= '3.8' From a11ddbff95e1bad7b4d4d2d4cc04941240ae7af8 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Thu, 14 Dec 2023 14:35:59 +0100 Subject: [PATCH 380/519] chore(deps): update all dependencies (#714) --- .../pandas-gbq/samples/snippets/requirements-test.txt | 4 ++-- packages/pandas-gbq/samples/snippets/requirements.txt | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements-test.txt b/packages/pandas-gbq/samples/snippets/requirements-test.txt index 00742565d64a..5d2f0c542249 100644 --- a/packages/pandas-gbq/samples/snippets/requirements-test.txt +++ b/packages/pandas-gbq/samples/snippets/requirements-test.txt @@ -1,2 +1,2 @@ -google-cloud-testutils==1.3.3 -pytest==7.4.1 +google-cloud-testutils==1.4.0 +pytest==7.4.3 diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index d26ba789bd40..616d4cdfb9b5 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,8 +1,8 @@ -google-cloud-bigquery-storage==2.22.0 -google-cloud-bigquery==3.11.4 -pandas-gbq==0.19.2 +google-cloud-bigquery-storage==2.24.0 +google-cloud-bigquery==3.14.1 +pandas-gbq==0.20.0 pandas===1.3.5; python_version == '3.7' pandas===2.0.3; python_version == '3.8' -pandas==2.1.0; python_version >= '3.9' +pandas==2.1.4; python_version >= '3.9' pyarrow==12.0.1; python_version == '3.7' pyarrow==14.0.1; python_version >= '3.8' From 26ef0280f8ad80da0933a401cd07a3496462da13 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 14 Dec 2023 19:23:40 -0500 Subject: [PATCH 381/519] build: update actions/upload-artifact and actions/download-artifact (#716) Source-Link: https://github.com/googleapis/synthtool/commit/280ddaed417057dfe5b1395731de07b7d09f5058 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:346ab2efb51649c5dde7756cbbdc60dd394852ba83b9bbffc292a63549f33c17 Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 ++-- packages/pandas-gbq/.github/workflows/unittest.yml | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 40bf99731959..9bee24097165 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:230f7fe8a0d2ed81a519cfc15c6bb11c5b46b9fb449b8b1219b3771bcb520ad2 -# created: 2023-12-09T15:16:25.430769578Z + digest: sha256:346ab2efb51649c5dde7756cbbdc60dd394852ba83b9bbffc292a63549f33c17 +# created: 2023-12-14T22:17:57.611773021Z diff --git a/packages/pandas-gbq/.github/workflows/unittest.yml b/packages/pandas-gbq/.github/workflows/unittest.yml index 344ac949aa74..46ec73050335 100644 --- a/packages/pandas-gbq/.github/workflows/unittest.yml +++ b/packages/pandas-gbq/.github/workflows/unittest.yml @@ -26,9 +26,9 @@ jobs: run: | nox -s unit-${{ matrix.python }} - name: Upload coverage results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: coverage-artifacts + name: coverage-artifact-${{ matrix.python }} path: .coverage-${{ matrix.python }} cover: @@ -47,11 +47,11 @@ jobs: python -m pip install --upgrade setuptools pip wheel python -m pip install coverage - name: Download coverage results - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: - name: coverage-artifacts path: .coverage-results/ - name: Report coverage results run: | - coverage combine .coverage-results/.coverage* + find .coverage-results -type f -name '*.zip' -exec unzip {} \; + coverage combine .coverage-results/**/.coverage* coverage report --show-missing --fail-under=96 From e4ebaa2692d4870965f9085ddc555aa8dc96be8f Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 16 Jan 2024 11:05:29 -0600 Subject: [PATCH 382/519] fix: update runtime check for min google-cloud-bigquery to 3.3.5 (#721) The minimum version of google-cloud-bigquery was updated to 3.3.5 in pandas-gbq version 0.18.0 (released November 2022). This change updates the runtime check in features.py to reflect that minimum version and removes some dead code for feature checks that are no longer relevant. --- packages/pandas-gbq/pandas_gbq/features.py | 46 +++--------- packages/pandas-gbq/pandas_gbq/gbq.py | 7 +- packages/pandas-gbq/pandas_gbq/load.py | 33 +++------ packages/pandas-gbq/setup.py | 9 +-- .../pandas-gbq/tests/system/test_read_gbq.py | 8 +-- .../pandas-gbq/tests/unit/test_features.py | 71 ------------------- packages/pandas-gbq/tests/unit/test_gbq.py | 18 ++--- packages/pandas-gbq/tests/unit/test_load.py | 17 +---- 8 files changed, 32 insertions(+), 177 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/features.py b/packages/pandas-gbq/pandas_gbq/features.py index 4b70a14adb17..d2fc33cbe0f9 100644 --- a/packages/pandas-gbq/pandas_gbq/features.py +++ b/packages/pandas-gbq/pandas_gbq/features.py @@ -5,11 +5,7 @@ """Module for checking dependency versions and supported features.""" # https://github.com/googleapis/python-bigquery/blob/master/CHANGELOG.md -BIGQUERY_MINIMUM_VERSION = "1.27.2" -BIGQUERY_ACCURATE_TIMESTAMP_VERSION = "2.6.0" -BIGQUERY_FROM_DATAFRAME_CSV_VERSION = "2.6.0" -BIGQUERY_SUPPORTS_BIGNUMERIC_VERSION = "2.10.0" -BIGQUERY_NO_DATE_AS_OBJECT_VERSION = "3.0.0dev" +BIGQUERY_MINIMUM_VERSION = "3.3.5" PANDAS_VERBOSITY_DEPRECATION_VERSION = "0.23.0" PANDAS_BOOLEAN_DTYPE_VERSION = "1.0.0" PANDAS_PARQUET_LOSSLESS_TIMESTAMP_VERSION = "1.1.0" @@ -31,9 +27,15 @@ def bigquery_installed_version(self): self._bigquery_installed_version = packaging.version.parse( google.cloud.bigquery.__version__ ) + return self._bigquery_installed_version + + def bigquery_try_import(self): + import google.cloud.bigquery + import packaging.version + bigquery_minimum_version = packaging.version.parse(BIGQUERY_MINIMUM_VERSION) - if self._bigquery_installed_version < bigquery_minimum_version: + if self.bigquery_installed_version < bigquery_minimum_version: raise ImportError( "pandas-gbq requires google-cloud-bigquery >= {0}, " "current version {1}".format( @@ -41,37 +43,7 @@ def bigquery_installed_version(self): ) ) - return self._bigquery_installed_version - - @property - def bigquery_has_accurate_timestamp(self): - import packaging.version - - min_version = packaging.version.parse(BIGQUERY_ACCURATE_TIMESTAMP_VERSION) - return self.bigquery_installed_version >= min_version - - @property - def bigquery_has_bignumeric(self): - import packaging.version - - min_version = packaging.version.parse(BIGQUERY_SUPPORTS_BIGNUMERIC_VERSION) - return self.bigquery_installed_version >= min_version - - @property - def bigquery_has_from_dataframe_with_csv(self): - import packaging.version - - bigquery_from_dataframe_version = packaging.version.parse( - BIGQUERY_FROM_DATAFRAME_CSV_VERSION - ) - return self.bigquery_installed_version >= bigquery_from_dataframe_version - - @property - def bigquery_needs_date_as_object(self): - import packaging.version - - max_version = packaging.version.parse(BIGQUERY_NO_DATE_AS_OBJECT_VERSION) - return self.bigquery_installed_version < max_version + return google.cloud.bigquery @property def pandas_installed_version(self): diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index dbb9e5b54708..d090e2872658 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -367,9 +367,9 @@ def sizeof_fmt(num, suffix="B"): def get_client(self): import google.api_core.client_info - from google.cloud import bigquery import pandas + bigquery = FEATURES.bigquery_try_import() client_info = google.api_core.client_info.ClientInfo( user_agent="pandas-{}".format(pandas.__version__) ) @@ -563,10 +563,6 @@ def _download_results( if max_results is not None: create_bqstorage_client = False - to_dataframe_kwargs = {} - if FEATURES.bigquery_needs_date_as_object: - to_dataframe_kwargs["date_as_object"] = True - try: schema_fields = [field.to_api_repr() for field in rows_iter.schema] conversion_dtypes = _bqschema_to_nullsafe_dtypes(schema_fields) @@ -575,7 +571,6 @@ def _download_results( dtypes=conversion_dtypes, progress_bar_type=progress_bar_type, create_bqstorage_client=create_bqstorage_client, - **to_dataframe_kwargs, ) except self.http_error as ex: self.process_http_error(ex) diff --git a/packages/pandas-gbq/pandas_gbq/load.py b/packages/pandas-gbq/pandas_gbq/load.py index bad99584284e..8243c7f3f04a 100644 --- a/packages/pandas-gbq/pandas_gbq/load.py +++ b/packages/pandas-gbq/pandas_gbq/load.py @@ -14,7 +14,6 @@ from google.cloud import bigquery from pandas_gbq import exceptions -from pandas_gbq.features import FEATURES import pandas_gbq.schema @@ -252,28 +251,16 @@ def load_chunks( # TODO: yield progress depending on result() with timeout return [0] elif api_method == "load_csv": - if FEATURES.bigquery_has_from_dataframe_with_csv: - return load_csv_from_dataframe( - client, - dataframe, - destination_table_ref, - write_disposition, - location, - chunksize, - schema, - billing_project=billing_project, - ) - else: - return load_csv_from_file( - client, - dataframe, - destination_table_ref, - write_disposition, - location, - chunksize, - schema, - billing_project=billing_project, - ) + return load_csv_from_dataframe( + client, + dataframe, + destination_table_ref, + write_disposition, + location, + chunksize, + schema, + billing_project=billing_project, + ) else: raise ValueError( f"Got unexpected api_method: {api_method!r}, expected one of 'load_parquet', 'load_csv'." diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index d0b16c2e71bf..2d09f41b13df 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -34,12 +34,9 @@ "google-api-core >= 2.10.2, <3.0.0dev", "google-auth >=2.13.0", "google-auth-oauthlib >=0.7.0", - # Require 1.27.* because it has a fix for out-of-bounds timestamps. See: - # https://github.com/googleapis/python-bigquery/pull/209 and - # https://github.com/googleapis/python-bigquery-pandas/issues/365 - # Exclude 2.4.* because it has a bug where waiting for the query can hang - # indefinitely. https://github.com/pydata/pandas-gbq/issues/343 - "google-cloud-bigquery >=3.3.5,<4.0.0dev,!=2.4.*", + # Please also update the minimum version in pandas_gbq/features.py to + # allow pandas-gbq to detect invalid package versions at runtime. + "google-cloud-bigquery >=3.3.5,<4.0.0dev", "google-cloud-bigquery-storage >=2.16.2,<3.0.0dev", "packaging >=20.0.0", ] diff --git a/packages/pandas-gbq/tests/system/test_read_gbq.py b/packages/pandas-gbq/tests/system/test_read_gbq.py index d57477b1148d..fada140b075c 100644 --- a/packages/pandas-gbq/tests/system/test_read_gbq.py +++ b/packages/pandas-gbq/tests/system/test_read_gbq.py @@ -454,10 +454,6 @@ def writable_table( ), ), id="bignumeric-normal-range", - marks=pytest.mark.skipif( - not FEATURES.bigquery_has_bignumeric, - reason="BIGNUMERIC not supported in this version of google-cloud-bigquery", - ), ), pytest.param( *QueryTestCase( @@ -538,9 +534,7 @@ def writable_table( ), } ), - use_bqstorage_apis={True, False} - if FEATURES.bigquery_has_accurate_timestamp - else {True}, + use_bqstorage_apis={True, False}, ), id="issue365-extreme-datetimes", ), diff --git a/packages/pandas-gbq/tests/unit/test_features.py b/packages/pandas-gbq/tests/unit/test_features.py index bfe2ea9b7463..8f17c78f2379 100644 --- a/packages/pandas-gbq/tests/unit/test_features.py +++ b/packages/pandas-gbq/tests/unit/test_features.py @@ -13,77 +13,6 @@ def fresh_bigquery_version(monkeypatch): monkeypatch.setattr(FEATURES, "_pandas_installed_version", None) -@pytest.mark.parametrize( - ["bigquery_version", "expected"], - [ - ("1.27.2", False), - ("1.99.100", False), - ("2.5.4", False), - ("2.6.0", True), - ("2.6.1", True), - ("2.12.0", True), - ], -) -def test_bigquery_has_accurate_timestamp(monkeypatch, bigquery_version, expected): - import google.cloud.bigquery - - monkeypatch.setattr(google.cloud.bigquery, "__version__", bigquery_version) - assert FEATURES.bigquery_has_accurate_timestamp == expected - - -@pytest.mark.parametrize( - ["bigquery_version", "expected"], - [ - ("1.27.2", False), - ("1.99.100", False), - ("2.9.999", False), - ("2.10.0", True), - ("2.12.0", True), - ("3.0.0", True), - ], -) -def test_bigquery_has_bignumeric(monkeypatch, bigquery_version, expected): - import google.cloud.bigquery - - monkeypatch.setattr(google.cloud.bigquery, "__version__", bigquery_version) - assert FEATURES.bigquery_has_bignumeric == expected - - -@pytest.mark.parametrize( - ["bigquery_version", "expected"], - [ - ("1.27.2", False), - ("1.99.100", False), - ("2.5.4", False), - ("2.6.0", True), - ("2.6.1", True), - ("2.12.0", True), - ], -) -def test_bigquery_has_from_dataframe_with_csv(monkeypatch, bigquery_version, expected): - import google.cloud.bigquery - - monkeypatch.setattr(google.cloud.bigquery, "__version__", bigquery_version) - assert FEATURES.bigquery_has_from_dataframe_with_csv == expected - - -@pytest.mark.parametrize( - ["bigquery_version", "expected"], - [ - ("1.27.2", True), - ("1.99.100", True), - ("2.12.0", True), - ("3.0.0", False), - ("3.1.0", False), - ], -) -def test_bigquery_needs_date_as_object(monkeypatch, bigquery_version, expected): - import google.cloud.bigquery - - monkeypatch.setattr(google.cloud.bigquery, "__version__", bigquery_version) - assert FEATURES.bigquery_needs_date_as_object == expected - - @pytest.mark.parametrize( ["pandas_version", "expected"], [ diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index ba6206865998..703acf27eada 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -732,19 +732,11 @@ def test_read_gbq_use_bqstorage_api( assert df is not None mock_list_rows = mock_bigquery_client.list_rows("dest", max_results=100) - if FEATURES.bigquery_needs_date_as_object: - mock_list_rows.to_dataframe.assert_called_once_with( - create_bqstorage_client=True, - dtypes=mock.ANY, - progress_bar_type=mock.ANY, - date_as_object=True, - ) - else: - mock_list_rows.to_dataframe.assert_called_once_with( - create_bqstorage_client=True, - dtypes=mock.ANY, - progress_bar_type=mock.ANY, - ) + mock_list_rows.to_dataframe.assert_called_once_with( + create_bqstorage_client=True, + dtypes=mock.ANY, + progress_bar_type=mock.ANY, + ) def test_read_gbq_calls_tqdm(mock_bigquery_client, mock_service_account_credentials): diff --git a/packages/pandas-gbq/tests/unit/test_load.py b/packages/pandas-gbq/tests/unit/test_load.py index 1d99d9b4186b..b34b13782b5e 100644 --- a/packages/pandas-gbq/tests/unit/test_load.py +++ b/packages/pandas-gbq/tests/unit/test_load.py @@ -8,7 +8,6 @@ import decimal from io import StringIO import textwrap -from unittest import mock import db_dtypes import numpy @@ -17,13 +16,10 @@ import pytest from pandas_gbq import exceptions -from pandas_gbq.features import FEATURES from pandas_gbq import load def load_method(bqclient, api_method): - if not FEATURES.bigquery_has_from_dataframe_with_csv and api_method == "load_csv": - return bqclient.load_table_from_file return bqclient.load_table_from_dataframe @@ -180,12 +176,10 @@ def test_load_csv_from_file_generates_schema(mock_bigquery_client): @pytest.mark.parametrize( - ["bigquery_has_from_dataframe_with_csv", "api_method"], - [(True, "load_parquet"), (True, "load_csv"), (False, "load_csv")], + ["api_method"], + [("load_parquet",), ("load_csv",)], ) -def test_load_chunks_omits_policy_tags( - monkeypatch, mock_bigquery_client, bigquery_has_from_dataframe_with_csv, api_method -): +def test_load_chunks_omits_policy_tags(monkeypatch, mock_bigquery_client, api_method): """Ensure that policyTags are omitted. We don't want to change the policyTags via a load job, as this can cause @@ -193,11 +187,6 @@ def test_load_chunks_omits_policy_tags( """ import google.cloud.bigquery - monkeypatch.setattr( - type(FEATURES), - "bigquery_has_from_dataframe_with_csv", - mock.PropertyMock(return_value=bigquery_has_from_dataframe_with_csv), - ) df = pandas.DataFrame({"col1": [1, 2, 3]}) destination = google.cloud.bigquery.TableReference.from_string( "my-project.my_dataset.my_table" From d287af27b98b13abc1c595292b01832b6a7d5378 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 16 Jan 2024 17:46:15 +0000 Subject: [PATCH 383/519] build(python): fix `docs` and `docfx` builds (#725) Source-Link: https://togithub.com/googleapis/synthtool/commit/fac8444edd5f5526e804c306b766a271772a3e2f Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:5ea6d0ab82c956b50962f91d94e206d3921537ae5fe1549ec5326381d8905cfa --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 6 +++--- packages/pandas-gbq/.kokoro/requirements.txt | 6 +++--- packages/pandas-gbq/noxfile.py | 20 ++++++++++++++++++- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 9bee24097165..d8a1bbca7179 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -1,4 +1,4 @@ -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:346ab2efb51649c5dde7756cbbdc60dd394852ba83b9bbffc292a63549f33c17 -# created: 2023-12-14T22:17:57.611773021Z + digest: sha256:5ea6d0ab82c956b50962f91d94e206d3921537ae5fe1549ec5326381d8905cfa +# created: 2024-01-15T16:32:08.142785673Z diff --git a/packages/pandas-gbq/.kokoro/requirements.txt b/packages/pandas-gbq/.kokoro/requirements.txt index e5c1ffca94b7..bb3d6ca38b14 100644 --- a/packages/pandas-gbq/.kokoro/requirements.txt +++ b/packages/pandas-gbq/.kokoro/requirements.txt @@ -263,9 +263,9 @@ jeepney==0.8.0 \ # via # keyring # secretstorage -jinja2==3.1.2 \ - --hash=sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852 \ - --hash=sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61 +jinja2==3.1.3 \ + --hash=sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa \ + --hash=sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90 # via gcp-releasetool keyring==24.2.0 \ --hash=sha256:4901caaf597bfd3bbd78c9a0c7c4c29fcd8310dab2cffefe749e916b6527acd6 \ diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index a0d5a94cf997..5a0d0ed819ac 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -375,7 +375,16 @@ def docs(session): session.install("-e", ".") session.install( - "sphinx==4.0.1", + # We need to pin to specific versions of the `sphinxcontrib-*` packages + # which still support sphinx 4.x. + # See https://github.com/googleapis/sphinx-docfx-yaml/issues/344 + # and https://github.com/googleapis/sphinx-docfx-yaml/issues/345. + "sphinxcontrib-applehelp==1.0.4", + "sphinxcontrib-devhelp==1.0.2", + "sphinxcontrib-htmlhelp==2.0.1", + "sphinxcontrib-qthelp==1.0.3", + "sphinxcontrib-serializinghtml==1.1.5", + "sphinx==4.5.0", "alabaster", "recommonmark", ) @@ -401,6 +410,15 @@ def docfx(session): session.install("-e", ".") session.install( + # We need to pin to specific versions of the `sphinxcontrib-*` packages + # which still support sphinx 4.x. + # See https://github.com/googleapis/sphinx-docfx-yaml/issues/344 + # and https://github.com/googleapis/sphinx-docfx-yaml/issues/345. + "sphinxcontrib-applehelp==1.0.4", + "sphinxcontrib-devhelp==1.0.2", + "sphinxcontrib-htmlhelp==2.0.1", + "sphinxcontrib-qthelp==1.0.3", + "sphinxcontrib-serializinghtml==1.1.5", "gcp-sphinx-docfx-yaml", "alabaster", "recommonmark", From 1ffb54eda0c5fe67e29d02d36145550894479be6 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 16 Jan 2024 13:52:16 -0600 Subject: [PATCH 384/519] refactor: move query and wait logic to separate module (#720) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This prepares the way for using the `query_and_wait` method built-in to the client library when available. Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - [ ] Make sure to open an issue as a [bug/issue](https://togithub.com/googleapis/python-bigquery-pandas/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea - [ ] Ensure the tests and linter pass - [ ] Code coverage does not decrease (if any source code was changed) - [ ] Appropriate docs were updated (if necessary) Towards #710 🦕 --- packages/pandas-gbq/pandas_gbq/exceptions.py | 7 + packages/pandas-gbq/pandas_gbq/gbq.py | 154 ++-------------- packages/pandas-gbq/pandas_gbq/query.py | 175 +++++++++++++++++++ packages/pandas-gbq/tests/unit/test_gbq.py | 64 +------ packages/pandas-gbq/tests/unit/test_query.py | 83 +++++++++ 5 files changed, 286 insertions(+), 197 deletions(-) create mode 100644 packages/pandas-gbq/pandas_gbq/query.py create mode 100644 packages/pandas-gbq/tests/unit/test_query.py diff --git a/packages/pandas-gbq/pandas_gbq/exceptions.py b/packages/pandas-gbq/pandas_gbq/exceptions.py index 1b4f6925097a..574b2dec3555 100644 --- a/packages/pandas-gbq/pandas_gbq/exceptions.py +++ b/packages/pandas-gbq/pandas_gbq/exceptions.py @@ -35,3 +35,10 @@ class PerformanceWarning(RuntimeWarning): Such warnings can occur when dependencies for the requested feature aren't up-to-date. """ + + +class QueryTimeout(ValueError): + """ + Raised when the query request exceeds the timeoutMs value specified in the + BigQuery configuration. + """ diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index d090e2872658..c54cf59218da 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -3,7 +3,6 @@ # license that can be found in the LICENSE file. import copy -import concurrent.futures from datetime import datetime import logging import re @@ -20,8 +19,9 @@ if typing.TYPE_CHECKING: # pragma: NO COVER import pandas -from pandas_gbq.exceptions import AccessDenied, GenericGBQException +from pandas_gbq.exceptions import GenericGBQException, QueryTimeout from pandas_gbq.features import FEATURES +import pandas_gbq.query import pandas_gbq.schema import pandas_gbq.timestamp @@ -74,8 +74,6 @@ class DatasetCreationError(ValueError): Raised when the create dataset method fails """ - pass - class InvalidColumnOrder(ValueError): """ @@ -84,8 +82,6 @@ class InvalidColumnOrder(ValueError): returned by BigQuery. """ - pass - class InvalidIndexColumn(ValueError): """ @@ -94,8 +90,6 @@ class InvalidIndexColumn(ValueError): returned by BigQuery. """ - pass - class InvalidPageToken(ValueError): """ @@ -103,8 +97,6 @@ class InvalidPageToken(ValueError): or returns a duplicate page token. """ - pass - class InvalidSchema(ValueError): """ @@ -127,17 +119,6 @@ class NotFoundException(ValueError): not be found. """ - pass - - -class QueryTimeout(ValueError): - """ - Raised when the query request exceeds the timeoutMs value specified in the - BigQuery configuration. - """ - - pass - class TableCreationError(ValueError): """ @@ -340,10 +321,6 @@ def __init__( self.client = self.get_client() self.use_bqstorage_api = use_bqstorage_api - # BQ Queries costs $5 per TB. First 1 TB per month is free - # see here for more: https://cloud.google.com/bigquery/pricing - self.query_price_for_TB = 5.0 / 2**40 # USD/TB - def _start_timer(self): self.start = time.time() @@ -355,16 +332,6 @@ def log_elapsed_seconds(self, prefix="Elapsed", postfix="s.", overlong=6): if sec > overlong: logger.info("{} {} {}".format(prefix, sec, postfix)) - # http://stackoverflow.com/questions/1094841/reusable-library-to-get-human-readable-version-of-file-size - @staticmethod - def sizeof_fmt(num, suffix="B"): - fmt = "%3.1f %s%s" - for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]: - if abs(num) < 1024.0: - return fmt % (num, unit, suffix) - num /= 1024.0 - return fmt % (num, "Y", suffix) - def get_client(self): import google.api_core.client_info import pandas @@ -421,46 +388,10 @@ def download_table( user_dtypes=dtypes, ) - def _wait_for_query_job(self, query_reply, timeout_ms): - """Wait for query to complete, pausing occasionally to update progress. - - Args: - query_reply (QueryJob): - A query job which has started. - - timeout_ms (Optional[int]): - How long to wait before cancelling the query. - """ - # Wait at most 10 seconds so we can show progress. - # TODO(https://github.com/googleapis/python-bigquery-pandas/issues/327): - # Include a tqdm progress bar here instead of a stream of log messages. - timeout_sec = 10.0 - if timeout_ms: - timeout_sec = min(timeout_sec, timeout_ms / 1000.0) - - while query_reply.state != "DONE": - self.log_elapsed_seconds(" Elapsed", "s. Waiting...") - - if timeout_ms and timeout_ms < self.get_elapsed_seconds() * 1000: - self.client.cancel_job( - query_reply.job_id, location=query_reply.location - ) - raise QueryTimeout("Query timeout: {} ms".format(timeout_ms)) - - try: - query_reply.result(timeout=timeout_sec) - except concurrent.futures.TimeoutError: - # Use our own timeout logic - pass - except self.http_error as ex: - self.process_http_error(ex) - def run_query(self, query, max_results=None, progress_bar_type=None, **kwargs): - from google.auth.exceptions import RefreshError from google.cloud import bigquery - import pandas - job_config = { + job_config_dict = { "query": { "useLegacySql": self.dialect == "legacy" @@ -470,74 +401,27 @@ def run_query(self, query, max_results=None, progress_bar_type=None, **kwargs): } config = kwargs.get("configuration") if config is not None: - job_config.update(config) - - self._start_timer() + job_config_dict.update(config) - try: - logger.debug("Requesting query... ") - query_reply = self.client.query( - query, - job_config=bigquery.QueryJobConfig.from_api_repr(job_config), - location=self.location, - project=self.project_id, - ) - logger.debug("Query running...") - except (RefreshError, ValueError) as ex: - if self.private_key: - raise AccessDenied( - f"The service account credentials are not valid: {ex}" - ) - else: - raise AccessDenied( - "The credentials have been revoked or expired, " - f"please re-run the application to re-authorize: {ex}" - ) - except self.http_error as ex: - self.process_http_error(ex) - - job_id = query_reply.job_id - logger.debug("Job ID: %s" % job_id) - - timeout_ms = job_config.get("jobTimeoutMs") or job_config["query"].get( - "timeoutMs" - ) + timeout_ms = job_config_dict.get("jobTimeoutMs") or job_config_dict[ + "query" + ].get("timeoutMs") timeout_ms = int(timeout_ms) if timeout_ms else None - self._wait_for_query_job(query_reply, timeout_ms) - if query_reply.cache_hit: - logger.debug("Query done.\nCache hit.\n") - else: - bytes_processed = query_reply.total_bytes_processed or 0 - bytes_billed = query_reply.total_bytes_billed or 0 - logger.debug( - "Query done.\nProcessed: {} Billed: {}".format( - self.sizeof_fmt(bytes_processed), - self.sizeof_fmt(bytes_billed), - ) - ) - logger.debug( - "Standard price: ${:,.2f} USD\n".format( - bytes_billed * self.query_price_for_TB - ) - ) + self._start_timer() + job_config = bigquery.QueryJobConfig.from_api_repr(job_config_dict) + rows_iter = pandas_gbq.query.query_and_wait( + self, + self.client, + query, + location=self.location, + project_id=self.project_id, + job_config=job_config, + max_results=max_results, + timeout_ms=timeout_ms, + ) dtypes = kwargs.get("dtypes") - - # Ensure destination is populated. - try: - query_reply.result() - except self.http_error as ex: - self.process_http_error(ex) - - # Avoid attempting to download results from DML queries, which have no - # destination. - if query_reply.destination is None: - return pandas.DataFrame() - - rows_iter = self.client.list_rows( - query_reply.destination, max_results=max_results - ) return self._download_results( rows_iter, max_results=max_results, diff --git a/packages/pandas-gbq/pandas_gbq/query.py b/packages/pandas-gbq/pandas_gbq/query.py new file mode 100644 index 000000000000..35d5592076bf --- /dev/null +++ b/packages/pandas-gbq/pandas_gbq/query.py @@ -0,0 +1,175 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +from __future__ import annotations + +import concurrent.futures +import logging +from typing import Optional + +from google.cloud import bigquery + +import pandas_gbq.exceptions + + +logger = logging.getLogger(__name__) + + +# On-demand BQ Queries costs $6.25 per TB. First 1 TB per month is free +# see here for more: https://cloud.google.com/bigquery/pricing +QUERY_PRICE_FOR_TB = 6.25 / 2**40 # USD/TB + + +# http://stackoverflow.com/questions/1094841/reusable-library-to-get-human-readable-version-of-file-size +def sizeof_fmt(num, suffix="B"): + fmt = "%3.1f %s%s" + for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]: + if abs(num) < 1024.0: + return fmt % (num, unit, suffix) + num /= 1024.0 + return fmt % (num, "Y", suffix) + + +def _wait_for_query_job( + connector, + client: bigquery.Client, + query_reply: bigquery.QueryJob, + timeout_ms: Optional[float], +): + """Wait for query to complete, pausing occasionally to update progress. + + Args: + connector (GbqConnector): + General pandas-gbq "connector" with helpers for stateful progress + logs and error raising. + + client (bigquery.Client): + A connection to BigQuery, used to make API requests. + + query_reply (QueryJob): + A query job which has started. + + timeout_ms (Optional[int]): + How long to wait before cancelling the query. + """ + # Wait at most 10 seconds so we can show progress. + # TODO(https://github.com/googleapis/python-bigquery-pandas/issues/327): + # Include a tqdm progress bar here instead of a stream of log messages. + timeout_sec = 10.0 + if timeout_ms: + timeout_sec = min(timeout_sec, timeout_ms / 1000.0) + + while query_reply.state != "DONE": + connector.log_elapsed_seconds(" Elapsed", "s. Waiting...") + + if timeout_ms and timeout_ms < connector.get_elapsed_seconds() * 1000: + client.cancel_job(query_reply.job_id, location=query_reply.location) + raise pandas_gbq.exceptions.QueryTimeout( + "Query timeout: {} ms".format(timeout_ms) + ) + + try: + query_reply.result(timeout=timeout_sec) + except concurrent.futures.TimeoutError: + # Use our own timeout logic + pass + except connector.http_error as ex: + connector.process_http_error(ex) + + +def query_and_wait( + connector, + client: bigquery.Client, + query: str, + *, + job_config: bigquery.QueryJobConfig, + location: Optional[str], + project_id: Optional[str], + max_results: Optional[int], + timeout_ms: Optional[int], +): + """Start a query and wait for it to complete. + + Args: + connector (GbqConnector): + General pandas-gbq "connector" with helpers for stateful progress + logs and error raising. + + client (bigquery.Client): + A connection to BigQuery, used to make API requests. + + query (str): + The text of the query to run. + + job_config (bigquery.QueryJobConfig): + Options for running the query. + + location (Optional[str]): + BigQuery location to run the query. Uses the default if not set. + + project (Optional[str]): + GCP project ID where to run the query. Uses the default if not set. + + max_results (Optional[int]): + Maximum number of rows in the result set. + + timeout_ms (Optional[int]): + How long to wait before cancelling the query. + + Returns: + bigquery.RowIterator: + Result iterator from which we can download the results in the + desired format (pandas.DataFrame). + """ + from google.auth.exceptions import RefreshError + + try: + logger.debug("Requesting query... ") + query_reply = client.query( + query, + job_config=job_config, + location=location, + project=project_id, + ) + logger.debug("Query running...") + except (RefreshError, ValueError) as ex: + if connector.private_key: + raise pandas_gbq.exceptions.AccessDenied( + f"The service account credentials are not valid: {ex}" + ) + else: + raise pandas_gbq.exceptions.AccessDenied( + "The credentials have been revoked or expired, " + f"please re-run the application to re-authorize: {ex}" + ) + except connector.http_error as ex: + connector.process_http_error(ex) + + job_id = query_reply.job_id + logger.debug("Job ID: %s" % job_id) + + _wait_for_query_job(connector, connector.client, query_reply, timeout_ms) + + if query_reply.cache_hit: + logger.debug("Query done.\nCache hit.\n") + else: + bytes_processed = query_reply.total_bytes_processed or 0 + bytes_billed = query_reply.total_bytes_billed or 0 + logger.debug( + "Query done.\nProcessed: {} Billed: {}".format( + sizeof_fmt(bytes_processed), + sizeof_fmt(bytes_billed), + ) + ) + logger.debug( + "Standard price: ${:,.2f} USD\n".format(bytes_billed * QUERY_PRICE_FOR_TB) + ) + + # As of google-cloud-bigquery 2.3.0, QueryJob.result() uses + # getQueryResults() instead of tabledata.list, which returns the correct + # response with DML/DDL queries. + try: + return query_reply.result(max_results=max_results) + except connector.http_error as ex: + connector.process_http_error(ex) diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index 703acf27eada..423589b150ce 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -4,12 +4,10 @@ # -*- coding: utf-8 -*- -import concurrent.futures import copy import datetime from unittest import mock -import freezegun import google.api_core.exceptions import numpy import pandas @@ -135,42 +133,6 @@ def test__transform_read_gbq_configuration_makes_copy(original, expected): assert did_change == should_change -def test__wait_for_query_job_exits_when_done(mock_bigquery_client): - connector = _make_connector() - connector.client = mock_bigquery_client - connector.start = datetime.datetime(2020, 1, 1).timestamp() - - mock_query = mock.create_autospec(google.cloud.bigquery.QueryJob) - type(mock_query).state = mock.PropertyMock(side_effect=("RUNNING", "DONE")) - mock_query.result.side_effect = concurrent.futures.TimeoutError("fake timeout") - - with freezegun.freeze_time("2020-01-01 00:00:00", tick=False): - connector._wait_for_query_job(mock_query, 60) - - mock_bigquery_client.cancel_job.assert_not_called() - - -def test__wait_for_query_job_cancels_after_timeout(mock_bigquery_client): - connector = _make_connector() - connector.client = mock_bigquery_client - connector.start = datetime.datetime(2020, 1, 1).timestamp() - - mock_query = mock.create_autospec(google.cloud.bigquery.QueryJob) - mock_query.job_id = "a-random-id" - mock_query.location = "job-location" - mock_query.state = "RUNNING" - mock_query.result.side_effect = concurrent.futures.TimeoutError("fake timeout") - - with freezegun.freeze_time( - "2020-01-01 00:00:00", auto_tick_seconds=15 - ), pytest.raises(gbq.QueryTimeout): - connector._wait_for_query_job(mock_query, 60) - - mock_bigquery_client.cancel_job.assert_called_with( - "a-random-id", location="job-location" - ) - - def test_GbqConnector_get_client_w_new_bq(mock_bigquery_client): gbq._test_google_api_imports() pytest.importorskip("google.api_core.client_info") @@ -519,10 +481,10 @@ def test_read_gbq_with_max_results_zero(monkeypatch): assert df is None -def test_read_gbq_with_max_results_ten(monkeypatch, mock_bigquery_client): +def test_read_gbq_with_max_results_ten(monkeypatch, mock_query_job): df = gbq.read_gbq("SELECT 1", dialect="standard", max_results=10) assert df is not None - mock_bigquery_client.list_rows.assert_called_with(mock.ANY, max_results=10) + mock_query_job.result.assert_called_with(max_results=10) @pytest.mark.parametrize(["verbose"], [(True,), (False,)]) @@ -825,28 +787,6 @@ def test_read_gbq_with_list_rows_error_translates_exception( ) -@pytest.mark.parametrize( - ["size_in_bytes", "formatted_text"], - [ - (999, "999.0 B"), - (1024, "1.0 KB"), - (1099, "1.1 KB"), - (1044480, "1020.0 KB"), - (1048576, "1.0 MB"), - (1048576000, "1000.0 MB"), - (1073741824, "1.0 GB"), - (1.099512e12, "1.0 TB"), - (1.125900e15, "1.0 PB"), - (1.152922e18, "1.0 EB"), - (1.180592e21, "1.0 ZB"), - (1.208926e24, "1.0 YB"), - (1.208926e28, "10000.0 YB"), - ], -) -def test_query_response_bytes(size_in_bytes, formatted_text): - assert gbq.GbqConnector.sizeof_fmt(size_in_bytes) == formatted_text - - def test_run_query_with_dml_query(mock_bigquery_client, mock_query_job): """ Don't attempt to download results from a DML query / query with no results. diff --git a/packages/pandas-gbq/tests/unit/test_query.py b/packages/pandas-gbq/tests/unit/test_query.py new file mode 100644 index 000000000000..b9a64535a92f --- /dev/null +++ b/packages/pandas-gbq/tests/unit/test_query.py @@ -0,0 +1,83 @@ +# Copyright (c) 2017 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +from __future__ import annotations + +import datetime +import concurrent.futures +from unittest import mock + +import freezegun +import google.cloud.bigquery +import pytest + +import pandas_gbq.exceptions +import pandas_gbq.gbq +import pandas_gbq.query as module_under_test + + +def _make_connector(project_id: str = "some-project", **kwargs): + return pandas_gbq.gbq.GbqConnector(project_id, **kwargs) + + +@pytest.mark.parametrize( + ["size_in_bytes", "formatted_text"], + [ + (999, "999.0 B"), + (1024, "1.0 KB"), + (1099, "1.1 KB"), + (1044480, "1020.0 KB"), + (1048576, "1.0 MB"), + (1048576000, "1000.0 MB"), + (1073741824, "1.0 GB"), + (1.099512e12, "1.0 TB"), + (1.125900e15, "1.0 PB"), + (1.152922e18, "1.0 EB"), + (1.180592e21, "1.0 ZB"), + (1.208926e24, "1.0 YB"), + (1.208926e28, "10000.0 YB"), + ], +) +def test_query_response_bytes(size_in_bytes, formatted_text): + assert module_under_test.sizeof_fmt(size_in_bytes) == formatted_text + + +def test__wait_for_query_job_exits_when_done(mock_bigquery_client): + connector = _make_connector() + connector.client = mock_bigquery_client + connector.start = datetime.datetime(2020, 1, 1).timestamp() + + mock_query = mock.create_autospec(google.cloud.bigquery.QueryJob) + type(mock_query).state = mock.PropertyMock(side_effect=("RUNNING", "DONE")) + mock_query.result.side_effect = concurrent.futures.TimeoutError("fake timeout") + + with freezegun.freeze_time("2020-01-01 00:00:00", tick=False): + module_under_test._wait_for_query_job( + connector, mock_bigquery_client, mock_query, 60 + ) + + mock_bigquery_client.cancel_job.assert_not_called() + + +def test__wait_for_query_job_cancels_after_timeout(mock_bigquery_client): + connector = _make_connector() + connector.client = mock_bigquery_client + connector.start = datetime.datetime(2020, 1, 1).timestamp() + + mock_query = mock.create_autospec(google.cloud.bigquery.QueryJob) + mock_query.job_id = "a-random-id" + mock_query.location = "job-location" + mock_query.state = "RUNNING" + mock_query.result.side_effect = concurrent.futures.TimeoutError("fake timeout") + + with freezegun.freeze_time( + "2020-01-01 00:00:00", auto_tick_seconds=15 + ), pytest.raises(pandas_gbq.exceptions.QueryTimeout): + module_under_test._wait_for_query_job( + connector, mock_bigquery_client, mock_query, 60 + ) + + mock_bigquery_client.cancel_job.assert_called_with( + "a-random-id", location="job-location" + ) From a122ec634cc91963aec5c681cdd065c37a289c9c Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 25 Jan 2024 17:01:44 -0600 Subject: [PATCH 385/519] feat: use faster query_and_wait method from google-cloud-bigquery when available (#722) * feat: use faster query_and_wait method from google-cloud-bigquery when available fix unit tests fix python 3.7 fix python 3.7 fix python 3.7 fix python 3.7 fix wait_timeout units boost test coverage remove dead code boost a little more coverage * restore missing test --- packages/pandas-gbq/pandas_gbq/features.py | 10 +- packages/pandas-gbq/pandas_gbq/gbq.py | 44 +++-- packages/pandas-gbq/pandas_gbq/query.py | 75 +++++--- .../pandas-gbq/tests/unit/test_context.py | 42 ++++- .../pandas-gbq/tests/unit/test_features.py | 17 ++ packages/pandas-gbq/tests/unit/test_gbq.py | 161 +++++++++++++++--- packages/pandas-gbq/tests/unit/test_query.py | 124 ++++++++++++++ 7 files changed, 411 insertions(+), 62 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/features.py b/packages/pandas-gbq/pandas_gbq/features.py index d2fc33cbe0f9..45a43c55f3f9 100644 --- a/packages/pandas-gbq/pandas_gbq/features.py +++ b/packages/pandas-gbq/pandas_gbq/features.py @@ -4,8 +4,9 @@ """Module for checking dependency versions and supported features.""" -# https://github.com/googleapis/python-bigquery/blob/master/CHANGELOG.md +# https://github.com/googleapis/python-bigquery/blob/main/CHANGELOG.md BIGQUERY_MINIMUM_VERSION = "3.3.5" +BIGQUERY_QUERY_AND_WAIT_VERSION = "3.14.0" PANDAS_VERBOSITY_DEPRECATION_VERSION = "0.23.0" PANDAS_BOOLEAN_DTYPE_VERSION = "1.0.0" PANDAS_PARQUET_LOSSLESS_TIMESTAMP_VERSION = "1.1.0" @@ -45,6 +46,13 @@ def bigquery_try_import(self): return google.cloud.bigquery + @property + def bigquery_has_query_and_wait(self): + import packaging.version + + min_version = packaging.version.parse(BIGQUERY_QUERY_AND_WAIT_VERSION) + return self.bigquery_installed_version >= min_version + @property def pandas_installed_version(self): import pandas diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index c54cf59218da..f93999a9118d 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -351,12 +351,17 @@ def process_http_error(ex): # See `BigQuery Troubleshooting Errors # `__ - if "cancelled" in ex.message: + message = ( + ex.message.casefold() + if hasattr(ex, "message") and ex.message is not None + else "" + ) + if "cancelled" in message: raise QueryTimeout("Reason: {0}".format(ex)) - elif "Provided Schema does not match" in ex.message: + elif "schema does not match" in message: error_message = ex.errors[0]["message"] raise InvalidSchema(f"Reason: {error_message}") - elif "Already Exists: Table" in ex.message: + elif "already exists: table" in message: error_message = ex.errors[0]["message"] raise TableCreationError(f"Reason: {error_message}") else: @@ -410,16 +415,29 @@ def run_query(self, query, max_results=None, progress_bar_type=None, **kwargs): self._start_timer() job_config = bigquery.QueryJobConfig.from_api_repr(job_config_dict) - rows_iter = pandas_gbq.query.query_and_wait( - self, - self.client, - query, - location=self.location, - project_id=self.project_id, - job_config=job_config, - max_results=max_results, - timeout_ms=timeout_ms, - ) + + if FEATURES.bigquery_has_query_and_wait: + rows_iter = pandas_gbq.query.query_and_wait_via_client_library( + self, + self.client, + query, + location=self.location, + project_id=self.project_id, + job_config=job_config, + max_results=max_results, + timeout_ms=timeout_ms, + ) + else: + rows_iter = pandas_gbq.query.query_and_wait( + self, + self.client, + query, + location=self.location, + project_id=self.project_id, + job_config=job_config, + max_results=max_results, + timeout_ms=timeout_ms, + ) dtypes = kwargs.get("dtypes") return self._download_results( diff --git a/packages/pandas-gbq/pandas_gbq/query.py b/packages/pandas-gbq/pandas_gbq/query.py index 35d5592076bf..0b7036f7a1b7 100644 --- a/packages/pandas-gbq/pandas_gbq/query.py +++ b/packages/pandas-gbq/pandas_gbq/query.py @@ -5,9 +5,11 @@ from __future__ import annotations import concurrent.futures +import functools import logging from typing import Optional +import google.auth.exceptions from google.cloud import bigquery import pandas_gbq.exceptions @@ -78,6 +80,26 @@ def _wait_for_query_job( connector.process_http_error(ex) +def try_query(connector, query_fn): + try: + logger.debug("Requesting query... ") + return query_fn() + except concurrent.futures.TimeoutError as ex: + raise pandas_gbq.exceptions.QueryTimeout("Reason: {0}".format(ex)) + except (google.auth.exceptions.RefreshError, ValueError) as ex: + if connector.private_key: + raise pandas_gbq.exceptions.AccessDenied( + f"The service account credentials are not valid: {ex}" + ) + else: + raise pandas_gbq.exceptions.AccessDenied( + "The credentials have been revoked or expired, " + f"please re-run the application to re-authorize: {ex}" + ) + except connector.http_error as ex: + connector.process_http_error(ex) + + def query_and_wait( connector, client: bigquery.Client, @@ -122,29 +144,17 @@ def query_and_wait( Result iterator from which we can download the results in the desired format (pandas.DataFrame). """ - from google.auth.exceptions import RefreshError - - try: - logger.debug("Requesting query... ") - query_reply = client.query( + query_reply = try_query( + connector, + functools.partial( + client.query, query, job_config=job_config, location=location, project=project_id, - ) - logger.debug("Query running...") - except (RefreshError, ValueError) as ex: - if connector.private_key: - raise pandas_gbq.exceptions.AccessDenied( - f"The service account credentials are not valid: {ex}" - ) - else: - raise pandas_gbq.exceptions.AccessDenied( - "The credentials have been revoked or expired, " - f"please re-run the application to re-authorize: {ex}" - ) - except connector.http_error as ex: - connector.process_http_error(ex) + ), + ) + logger.debug("Query running...") job_id = query_reply.job_id logger.debug("Job ID: %s" % job_id) @@ -173,3 +183,30 @@ def query_and_wait( return query_reply.result(max_results=max_results) except connector.http_error as ex: connector.process_http_error(ex) + + +def query_and_wait_via_client_library( + connector, + client: bigquery.Client, + query: str, + *, + job_config: bigquery.QueryJobConfig, + location: Optional[str], + project_id: Optional[str], + max_results: Optional[int], + timeout_ms: Optional[int], +): + rows_iter = try_query( + connector, + functools.partial( + client.query_and_wait, + query, + job_config=job_config, + location=location, + project=project_id, + max_results=max_results, + wait_timeout=timeout_ms / 1000.0 if timeout_ms else None, + ), + ) + logger.debug("Query done.\n") + return rows_iter diff --git a/packages/pandas-gbq/tests/unit/test_context.py b/packages/pandas-gbq/tests/unit/test_context.py index 1cf420f0cbca..6b6ce6a0744c 100644 --- a/packages/pandas-gbq/tests/unit/test_context.py +++ b/packages/pandas-gbq/tests/unit/test_context.py @@ -8,6 +8,7 @@ import google.cloud.bigquery import google.cloud.bigquery.table +import packaging.version import pytest @@ -55,8 +56,15 @@ def test_read_gbq_should_save_credentials(mock_get_credentials): mock_get_credentials.assert_not_called() -def test_read_gbq_should_use_dialect(mock_bigquery_client): +def test_read_gbq_should_use_dialect_with_query(monkeypatch, mock_bigquery_client): import pandas_gbq + import pandas_gbq.features + + monkeypatch.setattr( + pandas_gbq.features.FEATURES, + "_bigquery_installed_version", + packaging.version.parse(pandas_gbq.features.BIGQUERY_MINIMUM_VERSION), + ) assert pandas_gbq.context.dialect is None pandas_gbq.context.dialect = "legacy" @@ -71,3 +79,35 @@ def test_read_gbq_should_use_dialect(mock_bigquery_client): _, kwargs = mock_bigquery_client.query.call_args assert not kwargs["job_config"].use_legacy_sql pandas_gbq.context.dialect = None # Reset the global state. + + +def test_read_gbq_should_use_dialect_with_query_and_wait( + monkeypatch, mock_bigquery_client +): + if not hasattr(mock_bigquery_client, "query_and_wait"): + pytest.skip( + f"google-cloud-bigquery {google.cloud.bigquery.__version__} does not have query_and_wait" + ) + + import pandas_gbq + import pandas_gbq.features + + monkeypatch.setattr( + pandas_gbq.features.FEATURES, + "_bigquery_installed_version", + packaging.version.parse(pandas_gbq.features.BIGQUERY_QUERY_AND_WAIT_VERSION), + ) + + assert pandas_gbq.context.dialect is None + pandas_gbq.context.dialect = "legacy" + pandas_gbq.read_gbq("SELECT 1") + + _, kwargs = mock_bigquery_client.query_and_wait.call_args + assert kwargs["job_config"].use_legacy_sql + + pandas_gbq.context.dialect = "standard" + pandas_gbq.read_gbq("SELECT 1") + + _, kwargs = mock_bigquery_client.query_and_wait.call_args + assert not kwargs["job_config"].use_legacy_sql + pandas_gbq.context.dialect = None # Reset the global state. diff --git a/packages/pandas-gbq/tests/unit/test_features.py b/packages/pandas-gbq/tests/unit/test_features.py index 8f17c78f2379..947cb8ae50e7 100644 --- a/packages/pandas-gbq/tests/unit/test_features.py +++ b/packages/pandas-gbq/tests/unit/test_features.py @@ -13,6 +13,23 @@ def fresh_bigquery_version(monkeypatch): monkeypatch.setattr(FEATURES, "_pandas_installed_version", None) +@pytest.mark.parametrize( + ["bigquery_version", "expected"], + [ + ("1.99.100", False), + ("2.99.999", False), + ("3.13.11", False), + ("3.14.0", True), + ("4.999.999", True), + ], +) +def test_bigquery_has_query_and_wait(monkeypatch, bigquery_version, expected): + import google.cloud.bigquery + + monkeypatch.setattr(google.cloud.bigquery, "__version__", bigquery_version) + assert FEATURES.bigquery_has_query_and_wait == expected + + @pytest.mark.parametrize( ["pandas_version", "expected"], [ diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index 423589b150ce..ff8508c9c85f 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -9,12 +9,15 @@ from unittest import mock import google.api_core.exceptions +import google.cloud.bigquery import numpy +import packaging.version import pandas from pandas import DataFrame import pytest from pandas_gbq import gbq +import pandas_gbq.features from pandas_gbq.features import FEATURES @@ -40,29 +43,40 @@ def mock_query_job(): return mock_query -@pytest.fixture(autouse=True) -def default_bigquery_client(mock_bigquery_client, mock_query_job): +@pytest.fixture +def mock_row_iterator(mock_bigquery_client): mock_rows = mock.create_autospec(google.cloud.bigquery.table.RowIterator) mock_rows.total_rows = 1 - mock_rows.__iter__.return_value = [(1,)] - mock_query_job.result.return_value = mock_rows - mock_bigquery_client.list_rows.return_value = mock_rows - mock_bigquery_client.query.return_value = mock_query_job # Mock out SELECT 1 query results. def generate_schema(): - query = ( - mock_bigquery_client.query.call_args[0][0] - if mock_bigquery_client.query.call_args - else "" - ) + if mock_bigquery_client.query.call_args: + query = mock_bigquery_client.query.call_args[0][0] + elif ( + hasattr(mock_bigquery_client, "query_and_wait") + and mock_bigquery_client.query_and_wait.call_args + ): + query = mock_bigquery_client.query_and_wait.call_args[0][0] + else: + query = "" if query == "SELECT 1 AS int_col": return [google.cloud.bigquery.SchemaField("int_col", "INTEGER")] else: return [google.cloud.bigquery.SchemaField("_f0", "INTEGER")] type(mock_rows).schema = mock.PropertyMock(side_effect=generate_schema) + return mock_rows + + +@pytest.fixture(autouse=True) +def default_bigquery_client(mock_bigquery_client, mock_query_job, mock_row_iterator): + mock_query_job.result.return_value = mock_row_iterator + mock_bigquery_client.list_rows.return_value = mock_row_iterator + mock_bigquery_client.query.return_value = mock_query_job + + if hasattr(mock_bigquery_client, "query_and_wait"): + mock_bigquery_client.query_and_wait.return_value = mock_row_iterator # Mock out get_table. def get_table(table_ref_or_id, **kwargs): @@ -441,15 +455,51 @@ def test_read_gbq_with_no_project_id_given_should_fail(monkeypatch): gbq.read_gbq("SELECT 1", dialect="standard") -def test_read_gbq_with_inferred_project_id(mock_bigquery_client): +def test_read_gbq_with_inferred_project_id_with_query( + monkeypatch, mock_bigquery_client, mock_query_job +): + monkeypatch.setattr( + pandas_gbq.features.FEATURES, + "_bigquery_installed_version", + packaging.version.parse(pandas_gbq.features.BIGQUERY_MINIMUM_VERSION), + ) + type(mock_query_job).cache_hit = mock.PropertyMock(return_value=False) + type(mock_query_job).total_bytes_billed = mock.PropertyMock(return_value=10_000_000) + type(mock_query_job).total_bytes_processed = mock.PropertyMock(return_value=12345) + df = gbq.read_gbq("SELECT 1", dialect="standard") assert df is not None mock_bigquery_client.query.assert_called_once() -def test_read_gbq_with_inferred_project_id_from_service_account_credentials( - mock_bigquery_client, mock_service_account_credentials +def test_read_gbq_with_inferred_project_id_with_query_and_wait( + monkeypatch, mock_bigquery_client +): + if not hasattr(mock_bigquery_client, "query_and_wait"): + pytest.skip( + f"google-cloud-bigquery {google.cloud.bigquery.__version__} does not have query_and_wait" + ) + + monkeypatch.setattr( + pandas_gbq.features.FEATURES, + "_bigquery_installed_version", + packaging.version.parse(pandas_gbq.features.BIGQUERY_QUERY_AND_WAIT_VERSION), + ) + + df = gbq.read_gbq("SELECT 1", dialect="standard") + assert df is not None + mock_bigquery_client.query_and_wait.assert_called_once() + + +def test_read_gbq_with_inferred_project_id_from_service_account_credentials_with_query( + monkeypatch, mock_bigquery_client, mock_service_account_credentials ): + monkeypatch.setattr( + pandas_gbq.features.FEATURES, + "_bigquery_installed_version", + packaging.version.parse(pandas_gbq.features.BIGQUERY_MINIMUM_VERSION), + ) + mock_service_account_credentials.project_id = "service_account_project_id" df = gbq.read_gbq( "SELECT 1", @@ -465,6 +515,37 @@ def test_read_gbq_with_inferred_project_id_from_service_account_credentials( ) +def test_read_gbq_with_inferred_project_id_from_service_account_credentials_with_query_and_wait( + monkeypatch, mock_bigquery_client, mock_service_account_credentials +): + if not hasattr(mock_bigquery_client, "query_and_wait"): + pytest.skip( + f"google-cloud-bigquery {google.cloud.bigquery.__version__} does not have query_and_wait" + ) + + monkeypatch.setattr( + pandas_gbq.features.FEATURES, + "_bigquery_installed_version", + packaging.version.parse(pandas_gbq.features.BIGQUERY_QUERY_AND_WAIT_VERSION), + ) + + mock_service_account_credentials.project_id = "service_account_project_id" + df = gbq.read_gbq( + "SELECT 1", + dialect="standard", + credentials=mock_service_account_credentials, + ) + assert df is not None + mock_bigquery_client.query_and_wait.assert_called_once_with( + "SELECT 1", + job_config=mock.ANY, + location=None, + project="service_account_project_id", + max_results=None, + wait_timeout=None, + ) + + def test_read_gbq_without_inferred_project_id_from_compute_engine_credentials( mock_compute_engine_credentials, ): @@ -481,12 +562,42 @@ def test_read_gbq_with_max_results_zero(monkeypatch): assert df is None -def test_read_gbq_with_max_results_ten(monkeypatch, mock_query_job): +def test_read_gbq_with_max_results_ten_query(monkeypatch, mock_query_job): + monkeypatch.setattr( + pandas_gbq.features.FEATURES, + "_bigquery_installed_version", + packaging.version.parse(pandas_gbq.features.BIGQUERY_MINIMUM_VERSION), + ) df = gbq.read_gbq("SELECT 1", dialect="standard", max_results=10) assert df is not None mock_query_job.result.assert_called_with(max_results=10) +def test_read_gbq_with_max_results_ten_query_and_wait( + monkeypatch, mock_bigquery_client +): + if not hasattr(mock_bigquery_client, "query_and_wait"): + pytest.skip( + f"google-cloud-bigquery {google.cloud.bigquery.__version__} does not have query_and_wait" + ) + + monkeypatch.setattr( + pandas_gbq.features.FEATURES, + "_bigquery_installed_version", + packaging.version.parse(pandas_gbq.features.BIGQUERY_QUERY_AND_WAIT_VERSION), + ) + df = gbq.read_gbq("SELECT 1", dialect="standard", max_results=10) + assert df is not None + mock_bigquery_client.query_and_wait.assert_called_with( + "SELECT 1", + job_config=mock.ANY, + location=mock.ANY, + project=mock.ANY, + max_results=10, + wait_timeout=None, + ) + + @pytest.mark.parametrize(["verbose"], [(True,), (False,)]) def test_read_gbq_with_verbose_new_pandas_warns_deprecation(monkeypatch, verbose): monkeypatch.setattr( @@ -509,8 +620,6 @@ def test_read_gbq_wo_verbose_w_new_pandas_no_warnings(monkeypatch, recwarn): def test_read_gbq_with_old_bq_raises_importerror(monkeypatch): - import google.cloud.bigquery - monkeypatch.setattr(google.cloud.bigquery, "__version__", "0.27.0") monkeypatch.setattr(FEATURES, "_bigquery_installed_version", None) with pytest.raises(ImportError, match="google-cloud-bigquery"): @@ -665,7 +774,7 @@ def test_load_modifies_schema(mock_bigquery_client): assert new_schema == new_schema_cp -def test_read_gbq_passes_dtypes(mock_bigquery_client, mock_service_account_credentials): +def test_read_gbq_passes_dtypes(mock_service_account_credentials, mock_row_iterator): mock_service_account_credentials.project_id = "service_account_project_id" df = gbq.read_gbq( "SELECT 1 AS int_col", @@ -675,14 +784,13 @@ def test_read_gbq_passes_dtypes(mock_bigquery_client, mock_service_account_crede ) assert df is not None - mock_list_rows = mock_bigquery_client.list_rows("dest", max_results=100) - - _, to_dataframe_kwargs = mock_list_rows.to_dataframe.call_args + _, to_dataframe_kwargs = mock_row_iterator.to_dataframe.call_args assert to_dataframe_kwargs["dtypes"] == {"int_col": "my-custom-dtype"} def test_read_gbq_use_bqstorage_api( - mock_bigquery_client, mock_service_account_credentials + mock_service_account_credentials, + mock_row_iterator, ): mock_service_account_credentials.project_id = "service_account_project_id" df = gbq.read_gbq( @@ -693,15 +801,14 @@ def test_read_gbq_use_bqstorage_api( ) assert df is not None - mock_list_rows = mock_bigquery_client.list_rows("dest", max_results=100) - mock_list_rows.to_dataframe.assert_called_once_with( + mock_row_iterator.to_dataframe.assert_called_once_with( create_bqstorage_client=True, dtypes=mock.ANY, progress_bar_type=mock.ANY, ) -def test_read_gbq_calls_tqdm(mock_bigquery_client, mock_service_account_credentials): +def test_read_gbq_calls_tqdm(mock_service_account_credentials, mock_row_iterator): mock_service_account_credentials.project_id = "service_account_project_id" df = gbq.read_gbq( "SELECT 1", @@ -711,9 +818,7 @@ def test_read_gbq_calls_tqdm(mock_bigquery_client, mock_service_account_credenti ) assert df is not None - mock_list_rows = mock_bigquery_client.list_rows("dest", max_results=100) - - _, to_dataframe_kwargs = mock_list_rows.to_dataframe.call_args + _, to_dataframe_kwargs = mock_row_iterator.to_dataframe.call_args assert to_dataframe_kwargs["progress_bar_type"] == "foobar" diff --git a/packages/pandas-gbq/tests/unit/test_query.py b/packages/pandas-gbq/tests/unit/test_query.py index b9a64535a92f..5b63163496a8 100644 --- a/packages/pandas-gbq/tests/unit/test_query.py +++ b/packages/pandas-gbq/tests/unit/test_query.py @@ -9,6 +9,8 @@ from unittest import mock import freezegun +import google.api_core.exceptions +import google.auth.exceptions import google.cloud.bigquery import pytest @@ -21,6 +23,128 @@ def _make_connector(project_id: str = "some-project", **kwargs): return pandas_gbq.gbq.GbqConnector(project_id, **kwargs) +def test_query_and_wait_via_client_library_apierror_raises_genericgbqexception( + mock_bigquery_client, +): + if not hasattr(mock_bigquery_client, "query_and_wait"): + pytest.skip( + f"google-cloud-bigquery {google.cloud.bigquery.__version__} does not have query_and_wait" + ) + + connector = _make_connector() + connector.client = mock_bigquery_client + mock_bigquery_client.query_and_wait.side_effect = ( + google.api_core.exceptions.GoogleAPIError() + ) + + with pytest.raises(pandas_gbq.exceptions.GenericGBQException): + module_under_test.query_and_wait_via_client_library( + connector, + mock_bigquery_client, + "SELECT 1", + job_config=google.cloud.bigquery.QueryJobConfig(), + location=None, + project_id=None, + max_results=None, + timeout_ms=None, + ) + + +def test_query_and_wait_via_client_library_refresherror_raises_accessdenied_service_account( + mock_bigquery_client, +): + if not hasattr(mock_bigquery_client, "query_and_wait"): + pytest.skip( + f"google-cloud-bigquery {google.cloud.bigquery.__version__} does not have query_and_wait" + ) + + connector = _make_connector() + connector.private_key = "abc" + connector.client = mock_bigquery_client + mock_bigquery_client.query_and_wait.side_effect = ( + google.auth.exceptions.RefreshError() + ) + + with pytest.raises(pandas_gbq.exceptions.AccessDenied, match="service account"): + module_under_test.query_and_wait_via_client_library( + connector, + mock_bigquery_client, + "SELECT 1", + job_config=google.cloud.bigquery.QueryJobConfig(), + location=None, + project_id=None, + max_results=None, + timeout_ms=None, + ) + + +def test_query_and_wait_via_client_library_refresherror_raises_accessdenied_user_credentials( + mock_bigquery_client, +): + if not hasattr(mock_bigquery_client, "query_and_wait"): + pytest.skip( + f"google-cloud-bigquery {google.cloud.bigquery.__version__} does not have query_and_wait" + ) + + connector = _make_connector() + connector.client = mock_bigquery_client + mock_bigquery_client.query_and_wait.side_effect = ( + google.auth.exceptions.RefreshError() + ) + + with pytest.raises(pandas_gbq.exceptions.AccessDenied, match="revoked or expired"): + module_under_test.query_and_wait_via_client_library( + connector, + mock_bigquery_client, + "SELECT 1", + job_config=google.cloud.bigquery.QueryJobConfig(), + location=None, + project_id=None, + max_results=None, + timeout_ms=None, + ) + + +def test_query_and_wait_via_client_library_timeout_raises_querytimeout( + mock_bigquery_client, +): + if not hasattr(mock_bigquery_client, "query_and_wait"): + pytest.skip( + f"google-cloud-bigquery {google.cloud.bigquery.__version__} does not have query_and_wait" + ) + + connector = _make_connector() + connector.client = mock_bigquery_client + connector.start = datetime.datetime(2020, 1, 1).timestamp() + + mock_bigquery_client.query_and_wait.side_effect = concurrent.futures.TimeoutError( + "fake timeout" + ) + + with freezegun.freeze_time( + "2020-01-01 00:00:00", auto_tick_seconds=15 + ), pytest.raises(pandas_gbq.exceptions.QueryTimeout): + module_under_test.query_and_wait_via_client_library( + connector, + mock_bigquery_client, + "SELECT 1", + job_config=google.cloud.bigquery.QueryJobConfig(), + location="EU", + project_id="test-query-and-wait", + max_results=123, + timeout_ms=500, + ) + + mock_bigquery_client.query_and_wait.assert_called_with( + "SELECT 1", + job_config=mock.ANY, + location="EU", + project="test-query-and-wait", + max_results=123, + wait_timeout=0.5, + ) + + @pytest.mark.parametrize( ["size_in_bytes", "formatted_text"], [ From e86413637509dd9b0082984953dcaaee51cea845 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 5 Feb 2024 10:46:31 -0600 Subject: [PATCH 386/519] chore(main): release 0.21.0 (#726) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- packages/pandas-gbq/CHANGELOG.md | 12 ++++++++++++ packages/pandas-gbq/pandas_gbq/version.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index 9bbae45eed49..0ae335d31b63 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## [0.21.0](https://github.com/googleapis/python-bigquery-pandas/compare/v0.20.0...v0.21.0) (2024-01-25) + + +### Features + +* Use faster query_and_wait method from google-cloud-bigquery when available ([#722](https://github.com/googleapis/python-bigquery-pandas/issues/722)) ([ac3ce3f](https://github.com/googleapis/python-bigquery-pandas/commit/ac3ce3fcd1637fe741af0a10d765ba3092d0e668)) + + +### Bug Fixes + +* Update runtime check for min google-cloud-bigquery to 3.3.5 ([#721](https://github.com/googleapis/python-bigquery-pandas/issues/721)) ([b5f4869](https://github.com/googleapis/python-bigquery-pandas/commit/b5f48690334edae5f54373a3a8864e3c3496ff29)) + ## [0.20.0](https://github.com/googleapis/python-bigquery-pandas/compare/v0.19.2...v0.20.0) (2023-12-10) diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py index ed24fc9ea745..cc18b61f28f8 100644 --- a/packages/pandas-gbq/pandas_gbq/version.py +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -2,4 +2,4 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -__version__ = "0.20.0" +__version__ = "0.21.0" From 285e1e3a110e10492538557b605e9395e0ce61b6 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Wed, 7 Feb 2024 15:48:15 -0800 Subject: [PATCH 387/519] build(deps): bump cryptography from 41.0.6 to 42.0.0 in /synthtool/gcp/templates/python_library/.kokoro (#731) Source-Link: https://github.com/googleapis/synthtool/commit/e13b22b1f660c80e4c3e735a9177d2f16c4b8bdc Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:97b671488ad548ef783a452a9e1276ac10f144d5ae56d98cc4bf77ba504082b4 Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 +- packages/pandas-gbq/.kokoro/requirements.txt | 57 +++++++++++-------- 2 files changed, 35 insertions(+), 26 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index d8a1bbca7179..2aefd0e91175 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:5ea6d0ab82c956b50962f91d94e206d3921537ae5fe1549ec5326381d8905cfa -# created: 2024-01-15T16:32:08.142785673Z + digest: sha256:97b671488ad548ef783a452a9e1276ac10f144d5ae56d98cc4bf77ba504082b4 +# created: 2024-02-06T03:20:16.660474034Z diff --git a/packages/pandas-gbq/.kokoro/requirements.txt b/packages/pandas-gbq/.kokoro/requirements.txt index bb3d6ca38b14..8c11c9f3e9b6 100644 --- a/packages/pandas-gbq/.kokoro/requirements.txt +++ b/packages/pandas-gbq/.kokoro/requirements.txt @@ -93,30 +93,39 @@ colorlog==6.7.0 \ # via # gcp-docuploader # nox -cryptography==41.0.6 \ - --hash=sha256:068bc551698c234742c40049e46840843f3d98ad7ce265fd2bd4ec0d11306596 \ - --hash=sha256:0f27acb55a4e77b9be8d550d762b0513ef3fc658cd3eb15110ebbcbd626db12c \ - --hash=sha256:2132d5865eea673fe6712c2ed5fb4fa49dba10768bb4cc798345748380ee3660 \ - --hash=sha256:3288acccef021e3c3c10d58933f44e8602cf04dba96d9796d70d537bb2f4bbc4 \ - --hash=sha256:35f3f288e83c3f6f10752467c48919a7a94b7d88cc00b0668372a0d2ad4f8ead \ - --hash=sha256:398ae1fc711b5eb78e977daa3cbf47cec20f2c08c5da129b7a296055fbb22aed \ - --hash=sha256:422e3e31d63743855e43e5a6fcc8b4acab860f560f9321b0ee6269cc7ed70cc3 \ - --hash=sha256:48783b7e2bef51224020efb61b42704207dde583d7e371ef8fc2a5fb6c0aabc7 \ - --hash=sha256:4d03186af98b1c01a4eda396b137f29e4e3fb0173e30f885e27acec8823c1b09 \ - --hash=sha256:5daeb18e7886a358064a68dbcaf441c036cbdb7da52ae744e7b9207b04d3908c \ - --hash=sha256:60e746b11b937911dc70d164060d28d273e31853bb359e2b2033c9e93e6f3c43 \ - --hash=sha256:742ae5e9a2310e9dade7932f9576606836ed174da3c7d26bc3d3ab4bd49b9f65 \ - --hash=sha256:7e00fb556bda398b99b0da289ce7053639d33b572847181d6483ad89835115f6 \ - --hash=sha256:85abd057699b98fce40b41737afb234fef05c67e116f6f3650782c10862c43da \ - --hash=sha256:8efb2af8d4ba9dbc9c9dd8f04d19a7abb5b49eab1f3694e7b5a16a5fc2856f5c \ - --hash=sha256:ae236bb8760c1e55b7a39b6d4d32d2279bc6c7c8500b7d5a13b6fb9fc97be35b \ - --hash=sha256:afda76d84b053923c27ede5edc1ed7d53e3c9f475ebaf63c68e69f1403c405a8 \ - --hash=sha256:b27a7fd4229abef715e064269d98a7e2909ebf92eb6912a9603c7e14c181928c \ - --hash=sha256:b648fe2a45e426aaee684ddca2632f62ec4613ef362f4d681a9a6283d10e079d \ - --hash=sha256:c5a550dc7a3b50b116323e3d376241829fd326ac47bc195e04eb33a8170902a9 \ - --hash=sha256:da46e2b5df770070412c46f87bac0849b8d685c5f2679771de277a422c7d0b86 \ - --hash=sha256:f39812f70fc5c71a15aa3c97b2bbe213c3f2a460b79bd21c40d033bb34a9bf36 \ - --hash=sha256:ff369dd19e8fe0528b02e8df9f2aeb2479f89b1270d90f96a63500afe9af5cae +cryptography==42.0.0 \ + --hash=sha256:0a68bfcf57a6887818307600c3c0ebc3f62fbb6ccad2240aa21887cda1f8df1b \ + --hash=sha256:146e971e92a6dd042214b537a726c9750496128453146ab0ee8971a0299dc9bd \ + --hash=sha256:14e4b909373bc5bf1095311fa0f7fcabf2d1a160ca13f1e9e467be1ac4cbdf94 \ + --hash=sha256:206aaf42e031b93f86ad60f9f5d9da1b09164f25488238ac1dc488334eb5e221 \ + --hash=sha256:3005166a39b70c8b94455fdbe78d87a444da31ff70de3331cdec2c568cf25b7e \ + --hash=sha256:324721d93b998cb7367f1e6897370644751e5580ff9b370c0a50dc60a2003513 \ + --hash=sha256:33588310b5c886dfb87dba5f013b8d27df7ffd31dc753775342a1e5ab139e59d \ + --hash=sha256:35cf6ed4c38f054478a9df14f03c1169bb14bd98f0b1705751079b25e1cb58bc \ + --hash=sha256:3ca482ea80626048975360c8e62be3ceb0f11803180b73163acd24bf014133a0 \ + --hash=sha256:56ce0c106d5c3fec1038c3cca3d55ac320a5be1b44bf15116732d0bc716979a2 \ + --hash=sha256:5a217bca51f3b91971400890905a9323ad805838ca3fa1e202a01844f485ee87 \ + --hash=sha256:678cfa0d1e72ef41d48993a7be75a76b0725d29b820ff3cfd606a5b2b33fda01 \ + --hash=sha256:69fd009a325cad6fbfd5b04c711a4da563c6c4854fc4c9544bff3088387c77c0 \ + --hash=sha256:6cf9b76d6e93c62114bd19485e5cb003115c134cf9ce91f8ac924c44f8c8c3f4 \ + --hash=sha256:74f18a4c8ca04134d2052a140322002fef535c99cdbc2a6afc18a8024d5c9d5b \ + --hash=sha256:85f759ed59ffd1d0baad296e72780aa62ff8a71f94dc1ab340386a1207d0ea81 \ + --hash=sha256:87086eae86a700307b544625e3ba11cc600c3c0ef8ab97b0fda0705d6db3d4e3 \ + --hash=sha256:8814722cffcfd1fbd91edd9f3451b88a8f26a5fd41b28c1c9193949d1c689dc4 \ + --hash=sha256:8fedec73d590fd30c4e3f0d0f4bc961aeca8390c72f3eaa1a0874d180e868ddf \ + --hash=sha256:9515ea7f596c8092fdc9902627e51b23a75daa2c7815ed5aa8cf4f07469212ec \ + --hash=sha256:988b738f56c665366b1e4bfd9045c3efae89ee366ca3839cd5af53eaa1401bce \ + --hash=sha256:a2a8d873667e4fd2f34aedab02ba500b824692c6542e017075a2efc38f60a4c0 \ + --hash=sha256:bd7cf7a8d9f34cc67220f1195884151426ce616fdc8285df9054bfa10135925f \ + --hash=sha256:bdce70e562c69bb089523e75ef1d9625b7417c6297a76ac27b1b8b1eb51b7d0f \ + --hash=sha256:be14b31eb3a293fc6e6aa2807c8a3224c71426f7c4e3639ccf1a2f3ffd6df8c3 \ + --hash=sha256:be41b0c7366e5549265adf2145135dca107718fa44b6e418dc7499cfff6b4689 \ + --hash=sha256:c310767268d88803b653fffe6d6f2f17bb9d49ffceb8d70aed50ad45ea49ab08 \ + --hash=sha256:c58115384bdcfe9c7f644c72f10f6f42bed7cf59f7b52fe1bf7ae0a622b3a139 \ + --hash=sha256:c640b0ef54138fde761ec99a6c7dc4ce05e80420262c20fa239e694ca371d434 \ + --hash=sha256:ca20550bb590db16223eb9ccc5852335b48b8f597e2f6f0878bbfd9e7314eb17 \ + --hash=sha256:d97aae66b7de41cdf5b12087b5509e4e9805ed6f562406dfcf60e8481a9a28f8 \ + --hash=sha256:e9326ca78111e4c645f7e49cbce4ed2f3f85e17b61a563328c85a5208cf34440 # via # gcp-releasetool # secretstorage From c9456dfaabaecb5ceac97856e1a1226b9ff02d3a Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 20 Feb 2024 12:50:41 -0800 Subject: [PATCH 388/519] build(deps): bump cryptography from 42.0.0 to 42.0.2 in .kokoro (#733) Source-Link: https://github.com/googleapis/synthtool/commit/8d392a55db44b00b4a9b995318051e334eecdcf1 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:a0c4463fcfd9893fc172a3b3db2b6ac0c7b94ec6ad458c7dcea12d9693615ac3 Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 +- packages/pandas-gbq/.kokoro/requirements.txt | 66 +++++++++---------- 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 2aefd0e91175..51213ca00ee3 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:97b671488ad548ef783a452a9e1276ac10f144d5ae56d98cc4bf77ba504082b4 -# created: 2024-02-06T03:20:16.660474034Z + digest: sha256:a0c4463fcfd9893fc172a3b3db2b6ac0c7b94ec6ad458c7dcea12d9693615ac3 +# created: 2024-02-17T12:21:23.177926195Z diff --git a/packages/pandas-gbq/.kokoro/requirements.txt b/packages/pandas-gbq/.kokoro/requirements.txt index 8c11c9f3e9b6..f80bdcd62981 100644 --- a/packages/pandas-gbq/.kokoro/requirements.txt +++ b/packages/pandas-gbq/.kokoro/requirements.txt @@ -93,39 +93,39 @@ colorlog==6.7.0 \ # via # gcp-docuploader # nox -cryptography==42.0.0 \ - --hash=sha256:0a68bfcf57a6887818307600c3c0ebc3f62fbb6ccad2240aa21887cda1f8df1b \ - --hash=sha256:146e971e92a6dd042214b537a726c9750496128453146ab0ee8971a0299dc9bd \ - --hash=sha256:14e4b909373bc5bf1095311fa0f7fcabf2d1a160ca13f1e9e467be1ac4cbdf94 \ - --hash=sha256:206aaf42e031b93f86ad60f9f5d9da1b09164f25488238ac1dc488334eb5e221 \ - --hash=sha256:3005166a39b70c8b94455fdbe78d87a444da31ff70de3331cdec2c568cf25b7e \ - --hash=sha256:324721d93b998cb7367f1e6897370644751e5580ff9b370c0a50dc60a2003513 \ - --hash=sha256:33588310b5c886dfb87dba5f013b8d27df7ffd31dc753775342a1e5ab139e59d \ - --hash=sha256:35cf6ed4c38f054478a9df14f03c1169bb14bd98f0b1705751079b25e1cb58bc \ - --hash=sha256:3ca482ea80626048975360c8e62be3ceb0f11803180b73163acd24bf014133a0 \ - --hash=sha256:56ce0c106d5c3fec1038c3cca3d55ac320a5be1b44bf15116732d0bc716979a2 \ - --hash=sha256:5a217bca51f3b91971400890905a9323ad805838ca3fa1e202a01844f485ee87 \ - --hash=sha256:678cfa0d1e72ef41d48993a7be75a76b0725d29b820ff3cfd606a5b2b33fda01 \ - --hash=sha256:69fd009a325cad6fbfd5b04c711a4da563c6c4854fc4c9544bff3088387c77c0 \ - --hash=sha256:6cf9b76d6e93c62114bd19485e5cb003115c134cf9ce91f8ac924c44f8c8c3f4 \ - --hash=sha256:74f18a4c8ca04134d2052a140322002fef535c99cdbc2a6afc18a8024d5c9d5b \ - --hash=sha256:85f759ed59ffd1d0baad296e72780aa62ff8a71f94dc1ab340386a1207d0ea81 \ - --hash=sha256:87086eae86a700307b544625e3ba11cc600c3c0ef8ab97b0fda0705d6db3d4e3 \ - --hash=sha256:8814722cffcfd1fbd91edd9f3451b88a8f26a5fd41b28c1c9193949d1c689dc4 \ - --hash=sha256:8fedec73d590fd30c4e3f0d0f4bc961aeca8390c72f3eaa1a0874d180e868ddf \ - --hash=sha256:9515ea7f596c8092fdc9902627e51b23a75daa2c7815ed5aa8cf4f07469212ec \ - --hash=sha256:988b738f56c665366b1e4bfd9045c3efae89ee366ca3839cd5af53eaa1401bce \ - --hash=sha256:a2a8d873667e4fd2f34aedab02ba500b824692c6542e017075a2efc38f60a4c0 \ - --hash=sha256:bd7cf7a8d9f34cc67220f1195884151426ce616fdc8285df9054bfa10135925f \ - --hash=sha256:bdce70e562c69bb089523e75ef1d9625b7417c6297a76ac27b1b8b1eb51b7d0f \ - --hash=sha256:be14b31eb3a293fc6e6aa2807c8a3224c71426f7c4e3639ccf1a2f3ffd6df8c3 \ - --hash=sha256:be41b0c7366e5549265adf2145135dca107718fa44b6e418dc7499cfff6b4689 \ - --hash=sha256:c310767268d88803b653fffe6d6f2f17bb9d49ffceb8d70aed50ad45ea49ab08 \ - --hash=sha256:c58115384bdcfe9c7f644c72f10f6f42bed7cf59f7b52fe1bf7ae0a622b3a139 \ - --hash=sha256:c640b0ef54138fde761ec99a6c7dc4ce05e80420262c20fa239e694ca371d434 \ - --hash=sha256:ca20550bb590db16223eb9ccc5852335b48b8f597e2f6f0878bbfd9e7314eb17 \ - --hash=sha256:d97aae66b7de41cdf5b12087b5509e4e9805ed6f562406dfcf60e8481a9a28f8 \ - --hash=sha256:e9326ca78111e4c645f7e49cbce4ed2f3f85e17b61a563328c85a5208cf34440 +cryptography==42.0.2 \ + --hash=sha256:087887e55e0b9c8724cf05361357875adb5c20dec27e5816b653492980d20380 \ + --hash=sha256:09a77e5b2e8ca732a19a90c5bca2d124621a1edb5438c5daa2d2738bfeb02589 \ + --hash=sha256:130c0f77022b2b9c99d8cebcdd834d81705f61c68e91ddd614ce74c657f8b3ea \ + --hash=sha256:141e2aa5ba100d3788c0ad7919b288f89d1fe015878b9659b307c9ef867d3a65 \ + --hash=sha256:28cb2c41f131a5758d6ba6a0504150d644054fd9f3203a1e8e8d7ac3aea7f73a \ + --hash=sha256:2f9f14185962e6a04ab32d1abe34eae8a9001569ee4edb64d2304bf0d65c53f3 \ + --hash=sha256:320948ab49883557a256eab46149df79435a22d2fefd6a66fe6946f1b9d9d008 \ + --hash=sha256:36d4b7c4be6411f58f60d9ce555a73df8406d484ba12a63549c88bd64f7967f1 \ + --hash=sha256:3b15c678f27d66d247132cbf13df2f75255627bcc9b6a570f7d2fd08e8c081d2 \ + --hash=sha256:3dbd37e14ce795b4af61b89b037d4bc157f2cb23e676fa16932185a04dfbf635 \ + --hash=sha256:4383b47f45b14459cab66048d384614019965ba6c1a1a141f11b5a551cace1b2 \ + --hash=sha256:44c95c0e96b3cb628e8452ec060413a49002a247b2b9938989e23a2c8291fc90 \ + --hash=sha256:4b063d3413f853e056161eb0c7724822a9740ad3caa24b8424d776cebf98e7ee \ + --hash=sha256:52ed9ebf8ac602385126c9a2fe951db36f2cb0c2538d22971487f89d0de4065a \ + --hash=sha256:55d1580e2d7e17f45d19d3b12098e352f3a37fe86d380bf45846ef257054b242 \ + --hash=sha256:5ef9bc3d046ce83c4bbf4c25e1e0547b9c441c01d30922d812e887dc5f125c12 \ + --hash=sha256:5fa82a26f92871eca593b53359c12ad7949772462f887c35edaf36f87953c0e2 \ + --hash=sha256:61321672b3ac7aade25c40449ccedbc6db72c7f5f0fdf34def5e2f8b51ca530d \ + --hash=sha256:701171f825dcab90969596ce2af253143b93b08f1a716d4b2a9d2db5084ef7be \ + --hash=sha256:841ec8af7a8491ac76ec5a9522226e287187a3107e12b7d686ad354bb78facee \ + --hash=sha256:8a06641fb07d4e8f6c7dda4fc3f8871d327803ab6542e33831c7ccfdcb4d0ad6 \ + --hash=sha256:8e88bb9eafbf6a4014d55fb222e7360eef53e613215085e65a13290577394529 \ + --hash=sha256:a00aee5d1b6c20620161984f8ab2ab69134466c51f58c052c11b076715e72929 \ + --hash=sha256:a047682d324ba56e61b7ea7c7299d51e61fd3bca7dad2ccc39b72bd0118d60a1 \ + --hash=sha256:a7ef8dd0bf2e1d0a27042b231a3baac6883cdd5557036f5e8df7139255feaac6 \ + --hash=sha256:ad28cff53f60d99a928dfcf1e861e0b2ceb2bc1f08a074fdd601b314e1cc9e0a \ + --hash=sha256:b9097a208875fc7bbeb1286d0125d90bdfed961f61f214d3f5be62cd4ed8a446 \ + --hash=sha256:b97fe7d7991c25e6a31e5d5e795986b18fbbb3107b873d5f3ae6dc9a103278e9 \ + --hash=sha256:e0ec52ba3c7f1b7d813cd52649a5b3ef1fc0d433219dc8c93827c57eab6cf888 \ + --hash=sha256:ea2c3ffb662fec8bbbfce5602e2c159ff097a4631d96235fcf0fb00e59e3ece4 \ + --hash=sha256:fa3dec4ba8fb6e662770b74f62f1a0c7d4e37e25b58b2bf2c1be4c95372b4a33 \ + --hash=sha256:fbeb725c9dc799a574518109336acccaf1303c30d45c075c665c0793c2f79a7f # via # gcp-releasetool # secretstorage From 703ec2c5c560b9a0b18eee9b08c80343ae647d24 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Fri, 23 Feb 2024 12:28:46 -0500 Subject: [PATCH 389/519] feat: move bqstorage to extras and add debug capability (#735) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: move bqstorage to extras and add debug capability * added some debug tactics to understand the recwarn variable * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * more debuggery * changed length of recwarn to account for tqdm warning * clean up some testing detritus * removes two out of date tests --------- Co-authored-by: Owl Bot --- packages/pandas-gbq/noxfile.py | 3 +++ packages/pandas-gbq/setup.py | 4 ++- packages/pandas-gbq/tests/unit/test_gbq.py | 31 ---------------------- 3 files changed, 6 insertions(+), 32 deletions(-) diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 5a0d0ed819ac..48b6a4b01937 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -46,6 +46,7 @@ UNIT_TEST_LOCAL_DEPENDENCIES = [] UNIT_TEST_DEPENDENCIES = [] UNIT_TEST_EXTRAS = [ + "bqstorage", "tqdm", ] UNIT_TEST_EXTRAS_BY_PYTHON = { @@ -177,6 +178,8 @@ def default(session): ) install_unittest_dependencies(session, "-c", constraints_path) + session.run("python", "-m", "pip", "freeze") + # Run py.test against the unit tests. session.run( "py.test", diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index 2d09f41b13df..0e5a7bbe5c9a 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -37,10 +37,12 @@ # Please also update the minimum version in pandas_gbq/features.py to # allow pandas-gbq to detect invalid package versions at runtime. "google-cloud-bigquery >=3.3.5,<4.0.0dev", - "google-cloud-bigquery-storage >=2.16.2,<3.0.0dev", "packaging >=20.0.0", ] extras = { + "bqstorage": [ + "google-cloud-bigquery-storage >=2.16.2, <3.0.0dev", + ], "tqdm": "tqdm>=4.23.0", } diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index ff8508c9c85f..8ba81b6d5a3e 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -222,37 +222,6 @@ def test_to_gbq_with_verbose_new_pandas_warns_deprecation(monkeypatch, verbose): pass -def test_to_gbq_wo_verbose_w_new_pandas_no_warnings(monkeypatch, recwarn): - monkeypatch.setattr( - type(FEATURES), - "pandas_has_deprecated_verbose", - mock.PropertyMock(return_value=True), - ) - try: - gbq.to_gbq(DataFrame([[1]]), "dataset.tablename", project_id="my-project") - except gbq.TableCreationError: - pass - assert len(recwarn) == 0 - - -def test_to_gbq_with_verbose_old_pandas_no_warnings(monkeypatch, recwarn): - monkeypatch.setattr( - type(FEATURES), - "pandas_has_deprecated_verbose", - mock.PropertyMock(return_value=False), - ) - try: - gbq.to_gbq( - DataFrame([[1]]), - "dataset.tablename", - project_id="my-project", - verbose=True, - ) - except gbq.TableCreationError: - pass - assert len(recwarn) == 0 - - def test_to_gbq_with_private_key_raises_notimplementederror(): with pytest.raises(NotImplementedError, match="private_key"): gbq.to_gbq( From 332bd67c29b6a936f99b66f630c885015386b741 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Mon, 26 Feb 2024 19:35:00 -0500 Subject: [PATCH 390/519] fix: Remove python 3.7 due to end of life (EOL) (#737) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: removing Python 3.7 due to EOL * remove 3.7 from noxfile.py * remove 3.7 from owlbot.py * removed 3.7 from sample req.txt * removed constraints-3.7 * removed references to 3.7 in setup.py * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * adds lower end constraint for db-dtypes with Python 3.8 * removes two more references to 3.7 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * updates constraint * update constraints for 3.8 * remove protobuf as a direct dependency - test * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot --- .../.github/sync-repo-settings.yaml | 2 -- .../pandas-gbq/.github/workflows/unittest.yml | 2 +- packages/pandas-gbq/CONTRIBUTING.rst | 8 +++---- packages/pandas-gbq/noxfile.py | 6 +++--- packages/pandas-gbq/owlbot.py | 4 ++-- .../samples/snippets/requirements.txt | 2 -- packages/pandas-gbq/setup.py | 3 +-- .../pandas-gbq/testing/constraints-3.7.txt | 20 ------------------ .../pandas-gbq/testing/constraints-3.8.txt | 21 ++++++++++++++++++- 9 files changed, 30 insertions(+), 38 deletions(-) delete mode 100644 packages/pandas-gbq/testing/constraints-3.7.txt diff --git a/packages/pandas-gbq/.github/sync-repo-settings.yaml b/packages/pandas-gbq/.github/sync-repo-settings.yaml index a6cf85d3dc36..a570d4affa97 100644 --- a/packages/pandas-gbq/.github/sync-repo-settings.yaml +++ b/packages/pandas-gbq/.github/sync-repo-settings.yaml @@ -11,7 +11,6 @@ branchProtectionRules: - 'OwlBot Post Processor' - 'docs' - 'lint' - - 'unit (3.7)' - 'unit (3.8)' - 'unit (3.9)' - 'unit (3.10)' @@ -20,7 +19,6 @@ branchProtectionRules: - 'cover' - 'Kokoro' - 'Samples - Lint' - - 'Samples - Python 3.7' - 'Samples - Python 3.8' - 'Samples - Python 3.9' - 'Samples - Python 3.10' diff --git a/packages/pandas-gbq/.github/workflows/unittest.yml b/packages/pandas-gbq/.github/workflows/unittest.yml index 46ec73050335..3c11914b035e 100644 --- a/packages/pandas-gbq/.github/workflows/unittest.yml +++ b/packages/pandas-gbq/.github/workflows/unittest.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] + python: ['3.8', '3.9', '3.10', '3.11', '3.12'] steps: - name: Checkout uses: actions/checkout@v4 diff --git a/packages/pandas-gbq/CONTRIBUTING.rst b/packages/pandas-gbq/CONTRIBUTING.rst index 487deabb5afa..620763e35694 100644 --- a/packages/pandas-gbq/CONTRIBUTING.rst +++ b/packages/pandas-gbq/CONTRIBUTING.rst @@ -22,7 +22,7 @@ In order to add a feature: documentation. - The feature must work fully on the following CPython versions: - 3.7, 3.8, 3.9, 3.10, 3.11 and 3.12 on both UNIX and Windows. + 3.8, 3.9, 3.10, 3.11 and 3.12 on both UNIX and Windows. - The feature must not add unnecessary dependencies (where "unnecessary" is of course subjective, but new dependencies should @@ -148,7 +148,7 @@ Running System Tests .. note:: - System tests are only configured to run under Python 3.7, 3.8, 3.9, 3.10, 3.11 and 3.12. + System tests are only configured to run under Python 3.8, 3.9, 3.10, 3.11 and 3.12. For expediency, we do not run them in older versions of Python 3. This alone will not run the tests. You'll need to change some local @@ -221,14 +221,12 @@ Supported Python Versions We support: -- `Python 3.7`_ - `Python 3.8`_ - `Python 3.9`_ - `Python 3.10`_ - `Python 3.11`_ - `Python 3.12`_ -.. _Python 3.7: https://docs.python.org/3.7/ .. _Python 3.8: https://docs.python.org/3.8/ .. _Python 3.9: https://docs.python.org/3.9/ .. _Python 3.10: https://docs.python.org/3.10/ @@ -241,7 +239,7 @@ Supported versions can be found in our ``noxfile.py`` `config`_. .. _config: https://github.com/googleapis/python-bigquery-pandas/blob/main/noxfile.py -We also explicitly decided to support Python 3 beginning with version 3.7. +We also explicitly decided to support Python 3 beginning with version 3.8. Reasons for this include: - Encouraging use of newest versions of Python 3 diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 48b6a4b01937..6ee180ef4871 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -32,7 +32,7 @@ DEFAULT_PYTHON_VERSION = "3.8" -UNIT_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] +UNIT_TEST_PYTHON_VERSIONS = ["3.8", "3.9", "3.10", "3.11", "3.12"] UNIT_TEST_STANDARD_DEPENDENCIES = [ "mock", "asyncmock", @@ -58,7 +58,7 @@ UNIT_TEST_PYTHON_VERSIONS[-1], ] -SYSTEM_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] +SYSTEM_TEST_PYTHON_VERSIONS = ["3.8", "3.9", "3.10", "3.11", "3.12"] SYSTEM_TEST_STANDARD_DEPENDENCIES = [ "mock", "pytest", @@ -488,7 +488,7 @@ def prerelease_deps(session): session.install(*constraints_deps) prerel_deps = [ - "protobuf", + # "protobuf", # dependency of grpc "six", "googleapis-common-protos", diff --git a/packages/pandas-gbq/owlbot.py b/packages/pandas-gbq/owlbot.py index e6c59be4b041..fffbb827e456 100644 --- a/packages/pandas-gbq/owlbot.py +++ b/packages/pandas-gbq/owlbot.py @@ -35,8 +35,8 @@ } extras = ["tqdm"] templated_files = common.py_library( - unit_test_python_versions=["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"], - system_test_python_versions=["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"], + unit_test_python_versions=["3.8", "3.9", "3.10", "3.11", "3.12"], + system_test_python_versions=["3.8", "3.9", "3.10", "3.11", "3.12"], cov_level=96, unit_test_external_dependencies=["freezegun"], unit_test_extras=extras, diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 616d4cdfb9b5..8006ac85d863 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,8 +1,6 @@ google-cloud-bigquery-storage==2.24.0 google-cloud-bigquery==3.14.1 pandas-gbq==0.20.0 -pandas===1.3.5; python_version == '3.7' pandas===2.0.3; python_version == '3.8' pandas==2.1.4; python_version >= '3.9' -pyarrow==12.0.1; python_version == '3.7' pyarrow==14.0.1; python_version >= '3.8' diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index 0e5a7bbe5c9a..6f30eacdeb0b 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -85,7 +85,6 @@ "License :: OSI Approved :: BSD License", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", @@ -99,7 +98,7 @@ packages=packages, install_requires=dependencies, extras_require=extras, - python_requires=">=3.7", + python_requires=">=3.8", include_package_data=True, zip_safe=False, ) diff --git a/packages/pandas-gbq/testing/constraints-3.7.txt b/packages/pandas-gbq/testing/constraints-3.7.txt deleted file mode 100644 index 2a4141bd3c29..000000000000 --- a/packages/pandas-gbq/testing/constraints-3.7.txt +++ /dev/null @@ -1,20 +0,0 @@ -# This constraints file is used to check that lower bounds -# are correct in setup.py -# List *all* library dependencies and extras in this file. -# Pin the version to the lower bound. -# -# e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", -# Then this file should have foo==1.14.0 -db-dtypes==1.0.4 -google-api-core==2.10.2 -google-auth==2.13.0 -google-auth-oauthlib==0.7.0 -google-cloud-bigquery==3.3.5 -google-cloud-bigquery-storage==2.16.2 -numpy==1.16.6 -pandas==1.1.4 -pyarrow==3.0.0 -pydata-google-auth==1.5.0 -tqdm==4.23.0 -protobuf==3.19.5 -packaging==20.0.0 diff --git a/packages/pandas-gbq/testing/constraints-3.8.txt b/packages/pandas-gbq/testing/constraints-3.8.txt index 9c67e95ef27d..f77e0f2d4140 100644 --- a/packages/pandas-gbq/testing/constraints-3.8.txt +++ b/packages/pandas-gbq/testing/constraints-3.8.txt @@ -1 +1,20 @@ -numpy==1.17.5 +# This constraints file is used to check that lower bounds +# are correct in setup.py +# List *all* library dependencies and extras in this file. +# Pin the version to the lower bound. +# +# e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", +# Then this file should have foo==1.14.0 +# protobuf==3.19.5 +db-dtypes==1.0.4 +google-api-core==2.10.2 +google-auth==2.13.0 +google-auth-oauthlib==0.7.0 +google-cloud-bigquery==3.3.5 +google-cloud-bigquery-storage==2.16.2 +numpy==1.16.6 +pandas==1.1.4 +pyarrow==3.0.0 +pydata-google-auth==1.5.0 +tqdm==4.23.0 +packaging==20.0.0 \ No newline at end of file From abbdee4ba62ed41347c4305929c32fdb57a5ff2e Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Tue, 27 Feb 2024 02:48:17 +0100 Subject: [PATCH 391/519] chore(deps): update all dependencies (#717) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [![Mend Renovate](https://app.renovatebot.com/images/banner.svg)](https://renovatebot.com) This PR contains the following updates: | Package | Change | Age | Adoption | Passing | Confidence | |---|---|---|---|---|---| | [google-cloud-bigquery](https://togithub.com/googleapis/python-bigquery) | `==3.14.1` -> `==3.17.2` | [![age](https://developer.mend.io/api/mc/badges/age/pypi/google-cloud-bigquery/3.17.2?slim=true)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://developer.mend.io/api/mc/badges/adoption/pypi/google-cloud-bigquery/3.17.2?slim=true)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://developer.mend.io/api/mc/badges/compatibility/pypi/google-cloud-bigquery/3.14.1/3.17.2?slim=true)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://developer.mend.io/api/mc/badges/confidence/pypi/google-cloud-bigquery/3.14.1/3.17.2?slim=true)](https://docs.renovatebot.com/merge-confidence/) | | [pandas](https://pandas.pydata.org) ([source](https://togithub.com/pandas-dev/pandas)) | `==2.1.4` -> `==2.2.0` | [![age](https://developer.mend.io/api/mc/badges/age/pypi/pandas/2.2.0?slim=true)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://developer.mend.io/api/mc/badges/adoption/pypi/pandas/2.2.0?slim=true)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://developer.mend.io/api/mc/badges/compatibility/pypi/pandas/2.1.4/2.2.0?slim=true)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://developer.mend.io/api/mc/badges/confidence/pypi/pandas/2.1.4/2.2.0?slim=true)](https://docs.renovatebot.com/merge-confidence/) | | [pandas-gbq](https://togithub.com/googleapis/python-bigquery-pandas) | `==0.20.0` -> `==0.21.0` | [![age](https://developer.mend.io/api/mc/badges/age/pypi/pandas-gbq/0.21.0?slim=true)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://developer.mend.io/api/mc/badges/adoption/pypi/pandas-gbq/0.21.0?slim=true)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://developer.mend.io/api/mc/badges/compatibility/pypi/pandas-gbq/0.20.0/0.21.0?slim=true)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://developer.mend.io/api/mc/badges/confidence/pypi/pandas-gbq/0.20.0/0.21.0?slim=true)](https://docs.renovatebot.com/merge-confidence/) | | [pyarrow](https://arrow.apache.org/) ([source](https://togithub.com/apache/arrow)) | `==14.0.1` -> `==15.0.0` | [![age](https://developer.mend.io/api/mc/badges/age/pypi/pyarrow/15.0.0?slim=true)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://developer.mend.io/api/mc/badges/adoption/pypi/pyarrow/15.0.0?slim=true)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://developer.mend.io/api/mc/badges/compatibility/pypi/pyarrow/14.0.1/15.0.0?slim=true)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://developer.mend.io/api/mc/badges/confidence/pypi/pyarrow/14.0.1/15.0.0?slim=true)](https://docs.renovatebot.com/merge-confidence/) | | [pytest](https://docs.pytest.org/en/latest/) ([source](https://togithub.com/pytest-dev/pytest), [changelog](https://docs.pytest.org/en/stable/changelog.html)) | `==7.4.3` -> `==8.0.0` | [![age](https://developer.mend.io/api/mc/badges/age/pypi/pytest/8.0.0?slim=true)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://developer.mend.io/api/mc/badges/adoption/pypi/pytest/8.0.0?slim=true)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://developer.mend.io/api/mc/badges/compatibility/pypi/pytest/7.4.3/8.0.0?slim=true)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://developer.mend.io/api/mc/badges/confidence/pypi/pytest/7.4.3/8.0.0?slim=true)](https://docs.renovatebot.com/merge-confidence/) | --- ### Release Notes
googleapis/python-bigquery (google-cloud-bigquery) ### [`v3.17.2`](https://togithub.com/googleapis/python-bigquery/blob/HEAD/CHANGELOG.md#3172-2024-01-30) [Compare Source](https://togithub.com/googleapis/python-bigquery/compare/v3.17.1...v3.17.2) ##### Bug Fixes - Change load_table_from_json autodetect logic ([#​1804](https://togithub.com/googleapis/python-bigquery/issues/1804)) ([6249032](https://togithub.com/googleapis/python-bigquery/commit/62490325f64e5d66303d9218992e28ac5f21cb3f)) ##### Documentation - Update to use API ([#​1781](https://togithub.com/googleapis/python-bigquery/issues/1781)) ([81563b0](https://togithub.com/googleapis/python-bigquery/commit/81563b06298fe3a64be6a89b583c3d64758ca12a)) - Update `client_query_destination_table.py` sample to use `query_and_wait` ([#​1783](https://togithub.com/googleapis/python-bigquery/issues/1783)) ([68ebbe1](https://togithub.com/googleapis/python-bigquery/commit/68ebbe12d455ce8e9b1784fb11787c2fb842ef22)) - Update query_external_sheets_permanent_table.py to use query_and_wait API ([#​1778](https://togithub.com/googleapis/python-bigquery/issues/1778)) ([a7be88a](https://togithub.com/googleapis/python-bigquery/commit/a7be88adf8a480ee61aa79789cb53df1b79bb091)) - Update sample for query_to_arrow to use query_and_wait API ([#​1776](https://togithub.com/googleapis/python-bigquery/issues/1776)) ([dbf10de](https://togithub.com/googleapis/python-bigquery/commit/dbf10dee51a7635e9b98658f205ded2de087a06f)) - Update the query destination table legacy file to use query_and_wait API ([#​1775](https://togithub.com/googleapis/python-bigquery/issues/1775)) ([ef89f9e](https://togithub.com/googleapis/python-bigquery/commit/ef89f9e58c22b3af5a7757b69daa030116012350)) - Update to use `query_and_wait` in `client_query_w_positional_params.py` ([#​1786](https://togithub.com/googleapis/python-bigquery/issues/1786)) ([410f71e](https://togithub.com/googleapis/python-bigquery/commit/410f71e6b6e755928e363ed89c1044e14b0db9cc)) - Update to use `query_and_wait` in `samples/client_query_w_timestamp_params.py` ([#​1785](https://togithub.com/googleapis/python-bigquery/issues/1785)) ([ba36948](https://togithub.com/googleapis/python-bigquery/commit/ba3694852c13c8a29fe0f9d923353e82acfd4278)) - Update to_geodataframe to use query_and_wait functionality ([#​1800](https://togithub.com/googleapis/python-bigquery/issues/1800)) ([1298594](https://togithub.com/googleapis/python-bigquery/commit/12985942942b8f205ecd261fcdf620df9a640460)) ### [`v3.17.1`](https://togithub.com/googleapis/python-bigquery/blob/HEAD/CHANGELOG.md#3171-2024-01-24) [Compare Source](https://togithub.com/googleapis/python-bigquery/compare/v3.17.0...v3.17.1) ##### Bug Fixes - Add pyarrow.large_strign to the \_ARROW_SCALAR_IDS_TO_BQ map ([#​1796](https://togithub.com/googleapis/python-bigquery/issues/1796)) ([b402a6d](https://togithub.com/googleapis/python-bigquery/commit/b402a6df92e656aee10dd2c11c48f6ed93c74fd7)) - Retry 'job exceeded rate limits' for DDL queries ([#​1794](https://togithub.com/googleapis/python-bigquery/issues/1794)) ([39f33b2](https://togithub.com/googleapis/python-bigquery/commit/39f33b210ecbe9c2fd390825d29393c2d80257f5)) ### [`v3.17.0`](https://togithub.com/googleapis/python-bigquery/blob/HEAD/CHANGELOG.md#3170-2024-01-24) [Compare Source](https://togithub.com/googleapis/python-bigquery/compare/v3.16.0...v3.17.0) ##### Features - Support universe resolution ([#​1774](https://togithub.com/googleapis/python-bigquery/issues/1774)) ([0b5c1d5](https://togithub.com/googleapis/python-bigquery/commit/0b5c1d597cdec3a05a16fb935595f773c5840bd4)) ##### Bug Fixes - `query_and_wait` now retains unknown query configuration `_properties` ([#​1793](https://togithub.com/googleapis/python-bigquery/issues/1793)) ([4ba4342](https://togithub.com/googleapis/python-bigquery/commit/4ba434287a0a25f027e3b63a80f8881a9b16723e)) - Raise `ValueError` in `query_and_wait` with wrong `job_config` type ([4ba4342](https://togithub.com/googleapis/python-bigquery/commit/4ba434287a0a25f027e3b63a80f8881a9b16723e)) ##### Documentation - Remove unused query code sample ([#​1769](https://togithub.com/googleapis/python-bigquery/issues/1769)) ([1f96439](https://togithub.com/googleapis/python-bigquery/commit/1f96439b3dbd27f11be5e2af84f290ec6094d0a4)) - Update `snippets.py` to use `query_and_wait` ([#​1773](https://togithub.com/googleapis/python-bigquery/issues/1773)) ([d90602d](https://togithub.com/googleapis/python-bigquery/commit/d90602de87e58b665cb974401a327a640805822f)) - Update multiple samples to change query to query_and_wait ([#​1784](https://togithub.com/googleapis/python-bigquery/issues/1784)) ([d1161dd](https://togithub.com/googleapis/python-bigquery/commit/d1161dddde41a7d35b30033ccbf6984a5de640bd)) - Update the query with no cache sample to use query_and_wait API ([#​1770](https://togithub.com/googleapis/python-bigquery/issues/1770)) ([955a4cd](https://togithub.com/googleapis/python-bigquery/commit/955a4cd99e21cbca1b2f9c1dc6aa3fd8070cd61f)) - Updates `query` to `query and wait` in samples/desktopapp/user_credentials.py ([#​1787](https://togithub.com/googleapis/python-bigquery/issues/1787)) ([89f1299](https://togithub.com/googleapis/python-bigquery/commit/89f1299b3164b51fb0f29bc600a34ded59c10682)) ### [`v3.16.0`](https://togithub.com/googleapis/python-bigquery/blob/HEAD/CHANGELOG.md#3160-2024-01-12) [Compare Source](https://togithub.com/googleapis/python-bigquery/compare/v3.15.0...v3.16.0) ##### Features - Add `table_constraints` field to Table model ([#​1755](https://togithub.com/googleapis/python-bigquery/issues/1755)) ([a167f9a](https://togithub.com/googleapis/python-bigquery/commit/a167f9a95f0a8fbf0bdb4943d06f07c03768c132)) - Support jsonExtension in LoadJobConfig ([#​1751](https://togithub.com/googleapis/python-bigquery/issues/1751)) ([0fd7347](https://togithub.com/googleapis/python-bigquery/commit/0fd7347ddb4ae1993f02b3bc109f64297437b3e2)) ##### Bug Fixes - Add detailed message in job error ([#​1762](https://togithub.com/googleapis/python-bigquery/issues/1762)) ([08483fb](https://togithub.com/googleapis/python-bigquery/commit/08483fba675f3b87571787e1e4420134a8fc8177)) ### [`v3.15.0`](https://togithub.com/googleapis/python-bigquery/blob/HEAD/CHANGELOG.md#3150-2024-01-09) [Compare Source](https://togithub.com/googleapis/python-bigquery/compare/v3.14.1...v3.15.0) ##### Features - Support JSON type in `insert_rows` and as a scalar query parameter ([#​1757](https://togithub.com/googleapis/python-bigquery/issues/1757)) ([02a7d12](https://togithub.com/googleapis/python-bigquery/commit/02a7d129776b7da7da844ffa9c5cdf21811cd3af)) - Support RANGE in schema ([#​1746](https://togithub.com/googleapis/python-bigquery/issues/1746)) ([8585747](https://togithub.com/googleapis/python-bigquery/commit/8585747058e6db49a8078ae44d8e10735cdc27f9)) ##### Bug Fixes - Deserializing JSON subfields within structs fails ([#​1742](https://togithub.com/googleapis/python-bigquery/issues/1742)) ([0d93073](https://togithub.com/googleapis/python-bigquery/commit/0d930739c78b557db6cd48b38fe16eba93719c40)) - Due to upstream change in dataset, updates expected results ([#​1761](https://togithub.com/googleapis/python-bigquery/issues/1761)) ([132c14b](https://togithub.com/googleapis/python-bigquery/commit/132c14bbddfb61ea8bc408bef5e958e21b5b819c)) - Load_table_from_dataframe for higher scale decimal ([#​1703](https://togithub.com/googleapis/python-bigquery/issues/1703)) ([b9c8be0](https://togithub.com/googleapis/python-bigquery/commit/b9c8be0982c76187444300c414e0dda8b0ad105b)) - Updates types-protobuf version for mypy-samples nox session ([#​1764](https://togithub.com/googleapis/python-bigquery/issues/1764)) ([c0de695](https://togithub.com/googleapis/python-bigquery/commit/c0de6958e5761ad6ff532dd933b0f4387e18f1b9)) ##### Performance Improvements - DB-API uses more efficient `query_and_wait` when no job ID is provided ([#​1747](https://togithub.com/googleapis/python-bigquery/issues/1747)) ([d225a94](https://togithub.com/googleapis/python-bigquery/commit/d225a94e718a85877c495fbd32eca607b8919ac6))
pandas-dev/pandas (pandas) ### [`v2.2.0`](https://togithub.com/pandas-dev/pandas/compare/v2.1.4...v2.2.0) [Compare Source](https://togithub.com/pandas-dev/pandas/compare/v2.1.4...v2.2.0)
googleapis/python-bigquery-pandas (pandas-gbq) ### [`v0.21.0`](https://togithub.com/googleapis/python-bigquery-pandas/blob/HEAD/CHANGELOG.md#0210-2024-01-25) [Compare Source](https://togithub.com/googleapis/python-bigquery-pandas/compare/v0.20.0...v0.21.0) ##### Features - Use faster query_and_wait method from google-cloud-bigquery when available ([#​722](https://togithub.com/googleapis/python-bigquery-pandas/issues/722)) ([ac3ce3f](https://togithub.com/googleapis/python-bigquery-pandas/commit/ac3ce3fcd1637fe741af0a10d765ba3092d0e668)) ##### Bug Fixes - Update runtime check for min google-cloud-bigquery to 3.3.5 ([#​721](https://togithub.com/googleapis/python-bigquery-pandas/issues/721)) ([b5f4869](https://togithub.com/googleapis/python-bigquery-pandas/commit/b5f48690334edae5f54373a3a8864e3c3496ff29))
apache/arrow (pyarrow) ### [`v14.0.2`](https://togithub.com/apache/arrow/compare/apache-arrow-14.0.1...apache-arrow-14.0.2)
pytest-dev/pytest (pytest) ### [`v8.0.0`](https://togithub.com/pytest-dev/pytest/compare/7.4.4...8.0.0) [Compare Source](https://togithub.com/pytest-dev/pytest/compare/7.4.4...8.0.0) ### [`v7.4.4`](https://togithub.com/pytest-dev/pytest/compare/7.4.3...7.4.4) [Compare Source](https://togithub.com/pytest-dev/pytest/compare/7.4.3...7.4.4)
--- ### Configuration 📅 **Schedule**: Branch creation - At any time (no schedule defined), Automerge - At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 👻 **Immortal**: This PR will be recreated if closed unmerged. Get [config help](https://togithub.com/renovatebot/renovate/discussions) if that's undesired. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [Mend Renovate](https://www.mend.io/free-developer-tools/renovate/). View repository job log [here](https://developer.mend.io/github/googleapis/python-bigquery-pandas). --- .../pandas-gbq/samples/snippets/requirements-test.txt | 2 +- packages/pandas-gbq/samples/snippets/requirements.txt | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements-test.txt b/packages/pandas-gbq/samples/snippets/requirements-test.txt index 5d2f0c542249..0f7ae598a3d8 100644 --- a/packages/pandas-gbq/samples/snippets/requirements-test.txt +++ b/packages/pandas-gbq/samples/snippets/requirements-test.txt @@ -1,2 +1,2 @@ google-cloud-testutils==1.4.0 -pytest==7.4.3 +pytest==8.0.0 diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 8006ac85d863..b3248ca25594 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,6 +1,6 @@ google-cloud-bigquery-storage==2.24.0 -google-cloud-bigquery==3.14.1 -pandas-gbq==0.20.0 +google-cloud-bigquery==3.17.2 +pandas-gbq==0.21.0 pandas===2.0.3; python_version == '3.8' -pandas==2.1.4; python_version >= '3.9' -pyarrow==14.0.1; python_version >= '3.8' +pandas==2.2.0; python_version >= '3.9' +pyarrow==15.0.0; python_version >= '3.8' From dfa072db8cc72eb8c131f03a4faf93c74e1ef3b2 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Tue, 27 Feb 2024 05:29:57 +0100 Subject: [PATCH 392/519] chore(deps): update all dependencies (#738) --- packages/pandas-gbq/samples/snippets/requirements-test.txt | 2 +- packages/pandas-gbq/samples/snippets/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements-test.txt b/packages/pandas-gbq/samples/snippets/requirements-test.txt index 0f7ae598a3d8..271c772d8da7 100644 --- a/packages/pandas-gbq/samples/snippets/requirements-test.txt +++ b/packages/pandas-gbq/samples/snippets/requirements-test.txt @@ -1,2 +1,2 @@ google-cloud-testutils==1.4.0 -pytest==8.0.0 +pytest==8.0.2 diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index b3248ca25594..6874bbd58ce0 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -2,5 +2,5 @@ google-cloud-bigquery-storage==2.24.0 google-cloud-bigquery==3.17.2 pandas-gbq==0.21.0 pandas===2.0.3; python_version == '3.8' -pandas==2.2.0; python_version >= '3.9' +pandas==2.2.1; python_version >= '3.9' pyarrow==15.0.0; python_version >= '3.8' From 4c7bc0d1e1e3e6f8b5ec1add52ae00140d2168c9 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 27 Feb 2024 12:39:17 -0500 Subject: [PATCH 393/519] build(deps): bump cryptography from 42.0.2 to 42.0.4 in .kokoro (#739) Source-Link: https://github.com/googleapis/synthtool/commit/d895aec3679ad22aa120481f746bf9f2f325f26f Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:98f3afd11308259de6e828e37376d18867fd321aba07826e29e4f8d9cab56bad Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 +- packages/pandas-gbq/.kokoro/requirements.txt | 66 +++++++++---------- 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 51213ca00ee3..e4e943e0259a 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:a0c4463fcfd9893fc172a3b3db2b6ac0c7b94ec6ad458c7dcea12d9693615ac3 -# created: 2024-02-17T12:21:23.177926195Z + digest: sha256:98f3afd11308259de6e828e37376d18867fd321aba07826e29e4f8d9cab56bad +# created: 2024-02-27T15:56:18.442440378Z diff --git a/packages/pandas-gbq/.kokoro/requirements.txt b/packages/pandas-gbq/.kokoro/requirements.txt index f80bdcd62981..bda8e38c4f31 100644 --- a/packages/pandas-gbq/.kokoro/requirements.txt +++ b/packages/pandas-gbq/.kokoro/requirements.txt @@ -93,39 +93,39 @@ colorlog==6.7.0 \ # via # gcp-docuploader # nox -cryptography==42.0.2 \ - --hash=sha256:087887e55e0b9c8724cf05361357875adb5c20dec27e5816b653492980d20380 \ - --hash=sha256:09a77e5b2e8ca732a19a90c5bca2d124621a1edb5438c5daa2d2738bfeb02589 \ - --hash=sha256:130c0f77022b2b9c99d8cebcdd834d81705f61c68e91ddd614ce74c657f8b3ea \ - --hash=sha256:141e2aa5ba100d3788c0ad7919b288f89d1fe015878b9659b307c9ef867d3a65 \ - --hash=sha256:28cb2c41f131a5758d6ba6a0504150d644054fd9f3203a1e8e8d7ac3aea7f73a \ - --hash=sha256:2f9f14185962e6a04ab32d1abe34eae8a9001569ee4edb64d2304bf0d65c53f3 \ - --hash=sha256:320948ab49883557a256eab46149df79435a22d2fefd6a66fe6946f1b9d9d008 \ - --hash=sha256:36d4b7c4be6411f58f60d9ce555a73df8406d484ba12a63549c88bd64f7967f1 \ - --hash=sha256:3b15c678f27d66d247132cbf13df2f75255627bcc9b6a570f7d2fd08e8c081d2 \ - --hash=sha256:3dbd37e14ce795b4af61b89b037d4bc157f2cb23e676fa16932185a04dfbf635 \ - --hash=sha256:4383b47f45b14459cab66048d384614019965ba6c1a1a141f11b5a551cace1b2 \ - --hash=sha256:44c95c0e96b3cb628e8452ec060413a49002a247b2b9938989e23a2c8291fc90 \ - --hash=sha256:4b063d3413f853e056161eb0c7724822a9740ad3caa24b8424d776cebf98e7ee \ - --hash=sha256:52ed9ebf8ac602385126c9a2fe951db36f2cb0c2538d22971487f89d0de4065a \ - --hash=sha256:55d1580e2d7e17f45d19d3b12098e352f3a37fe86d380bf45846ef257054b242 \ - --hash=sha256:5ef9bc3d046ce83c4bbf4c25e1e0547b9c441c01d30922d812e887dc5f125c12 \ - --hash=sha256:5fa82a26f92871eca593b53359c12ad7949772462f887c35edaf36f87953c0e2 \ - --hash=sha256:61321672b3ac7aade25c40449ccedbc6db72c7f5f0fdf34def5e2f8b51ca530d \ - --hash=sha256:701171f825dcab90969596ce2af253143b93b08f1a716d4b2a9d2db5084ef7be \ - --hash=sha256:841ec8af7a8491ac76ec5a9522226e287187a3107e12b7d686ad354bb78facee \ - --hash=sha256:8a06641fb07d4e8f6c7dda4fc3f8871d327803ab6542e33831c7ccfdcb4d0ad6 \ - --hash=sha256:8e88bb9eafbf6a4014d55fb222e7360eef53e613215085e65a13290577394529 \ - --hash=sha256:a00aee5d1b6c20620161984f8ab2ab69134466c51f58c052c11b076715e72929 \ - --hash=sha256:a047682d324ba56e61b7ea7c7299d51e61fd3bca7dad2ccc39b72bd0118d60a1 \ - --hash=sha256:a7ef8dd0bf2e1d0a27042b231a3baac6883cdd5557036f5e8df7139255feaac6 \ - --hash=sha256:ad28cff53f60d99a928dfcf1e861e0b2ceb2bc1f08a074fdd601b314e1cc9e0a \ - --hash=sha256:b9097a208875fc7bbeb1286d0125d90bdfed961f61f214d3f5be62cd4ed8a446 \ - --hash=sha256:b97fe7d7991c25e6a31e5d5e795986b18fbbb3107b873d5f3ae6dc9a103278e9 \ - --hash=sha256:e0ec52ba3c7f1b7d813cd52649a5b3ef1fc0d433219dc8c93827c57eab6cf888 \ - --hash=sha256:ea2c3ffb662fec8bbbfce5602e2c159ff097a4631d96235fcf0fb00e59e3ece4 \ - --hash=sha256:fa3dec4ba8fb6e662770b74f62f1a0c7d4e37e25b58b2bf2c1be4c95372b4a33 \ - --hash=sha256:fbeb725c9dc799a574518109336acccaf1303c30d45c075c665c0793c2f79a7f +cryptography==42.0.4 \ + --hash=sha256:01911714117642a3f1792c7f376db572aadadbafcd8d75bb527166009c9f1d1b \ + --hash=sha256:0e89f7b84f421c56e7ff69f11c441ebda73b8a8e6488d322ef71746224c20fce \ + --hash=sha256:12d341bd42cdb7d4937b0cabbdf2a94f949413ac4504904d0cdbdce4a22cbf88 \ + --hash=sha256:15a1fb843c48b4a604663fa30af60818cd28f895572386e5f9b8a665874c26e7 \ + --hash=sha256:1cdcdbd117681c88d717437ada72bdd5be9de117f96e3f4d50dab3f59fd9ab20 \ + --hash=sha256:1df6fcbf60560d2113b5ed90f072dc0b108d64750d4cbd46a21ec882c7aefce9 \ + --hash=sha256:3c6048f217533d89f2f8f4f0fe3044bf0b2090453b7b73d0b77db47b80af8dff \ + --hash=sha256:3e970a2119507d0b104f0a8e281521ad28fc26f2820687b3436b8c9a5fcf20d1 \ + --hash=sha256:44a64043f743485925d3bcac548d05df0f9bb445c5fcca6681889c7c3ab12764 \ + --hash=sha256:4e36685cb634af55e0677d435d425043967ac2f3790ec652b2b88ad03b85c27b \ + --hash=sha256:5f8907fcf57392cd917892ae83708761c6ff3c37a8e835d7246ff0ad251d9298 \ + --hash=sha256:69b22ab6506a3fe483d67d1ed878e1602bdd5912a134e6202c1ec672233241c1 \ + --hash=sha256:6bfadd884e7280df24d26f2186e4e07556a05d37393b0f220a840b083dc6a824 \ + --hash=sha256:6d0fbe73728c44ca3a241eff9aefe6496ab2656d6e7a4ea2459865f2e8613257 \ + --hash=sha256:6ffb03d419edcab93b4b19c22ee80c007fb2d708429cecebf1dd3258956a563a \ + --hash=sha256:810bcf151caefc03e51a3d61e53335cd5c7316c0a105cc695f0959f2c638b129 \ + --hash=sha256:831a4b37accef30cccd34fcb916a5d7b5be3cbbe27268a02832c3e450aea39cb \ + --hash=sha256:887623fe0d70f48ab3f5e4dbf234986b1329a64c066d719432d0698522749929 \ + --hash=sha256:a0298bdc6e98ca21382afe914c642620370ce0470a01e1bef6dd9b5354c36854 \ + --hash=sha256:a1327f280c824ff7885bdeef8578f74690e9079267c1c8bd7dc5cc5aa065ae52 \ + --hash=sha256:c1f25b252d2c87088abc8bbc4f1ecbf7c919e05508a7e8628e6875c40bc70923 \ + --hash=sha256:c3a5cbc620e1e17009f30dd34cb0d85c987afd21c41a74352d1719be33380885 \ + --hash=sha256:ce8613beaffc7c14f091497346ef117c1798c202b01153a8cc7b8e2ebaaf41c0 \ + --hash=sha256:d2a27aca5597c8a71abbe10209184e1a8e91c1fd470b5070a2ea60cafec35bcd \ + --hash=sha256:dad9c385ba8ee025bb0d856714f71d7840020fe176ae0229de618f14dae7a6e2 \ + --hash=sha256:db4b65b02f59035037fde0998974d84244a64c3265bdef32a827ab9b63d61b18 \ + --hash=sha256:e09469a2cec88fb7b078e16d4adec594414397e8879a4341c6ace96013463d5b \ + --hash=sha256:e53dc41cda40b248ebc40b83b31516487f7db95ab8ceac1f042626bc43a2f992 \ + --hash=sha256:f1e85a178384bf19e36779d91ff35c7617c885da487d689b05c1366f9933ad74 \ + --hash=sha256:f47be41843200f7faec0683ad751e5ef11b9a56a220d57f300376cd8aba81660 \ + --hash=sha256:fb0cef872d8193e487fc6bdb08559c3aa41b659a7d9be48b2e10747f47863925 \ + --hash=sha256:ffc73996c4fca3d2b6c1c8c12bfd3ad00def8621da24f547626bf06441400449 # via # gcp-releasetool # secretstorage From 067617988a925c68b7e1b3f99e77477127406588 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Tue, 5 Mar 2024 22:57:34 +0100 Subject: [PATCH 394/519] chore(deps): update dependency google-cloud-bigquery to v3.18.0 (#740) --- packages/pandas-gbq/samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 6874bbd58ce0..d244f825c590 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,5 +1,5 @@ google-cloud-bigquery-storage==2.24.0 -google-cloud-bigquery==3.17.2 +google-cloud-bigquery==3.18.0 pandas-gbq==0.21.0 pandas===2.0.3; python_version == '3.8' pandas==2.2.1; python_version >= '3.9' From a282c29ddc6cf886a6a41e04d5afa011a9544f1a Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Thu, 7 Mar 2024 12:02:41 -0500 Subject: [PATCH 395/519] chore(main): release 0.22.0 (#736) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- packages/pandas-gbq/CHANGELOG.md | 12 ++++++++++++ packages/pandas-gbq/pandas_gbq/version.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index 0ae335d31b63..d115e541691b 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## [0.22.0](https://github.com/googleapis/python-bigquery-pandas/compare/v0.21.0...v0.22.0) (2024-03-05) + + +### Features + +* Move bqstorage to extras and add debug capability ([#735](https://github.com/googleapis/python-bigquery-pandas/issues/735)) ([366cb55](https://github.com/googleapis/python-bigquery-pandas/commit/366cb55a5f2dde4f92994de9ba6c59eb3e1d7c9f)) + + +### Bug Fixes + +* Remove python 3.7 due to end of life (EOL) ([#737](https://github.com/googleapis/python-bigquery-pandas/issues/737)) ([d0810e8](https://github.com/googleapis/python-bigquery-pandas/commit/d0810e82322c5cc81b33894b7e7f50c140f542ed)) + ## [0.21.0](https://github.com/googleapis/python-bigquery-pandas/compare/v0.20.0...v0.21.0) (2024-01-25) diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py index cc18b61f28f8..6f21e37d7272 100644 --- a/packages/pandas-gbq/pandas_gbq/version.py +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -2,4 +2,4 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -__version__ = "0.21.0" +__version__ = "0.22.0" From ac4be2cf5fb6410085b8609c9ec819332a2646cd Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Fri, 8 Mar 2024 00:57:40 +0100 Subject: [PATCH 396/519] chore(deps): update all dependencies (#741) --- packages/pandas-gbq/samples/snippets/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index d244f825c590..f632e34569ec 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,6 +1,6 @@ google-cloud-bigquery-storage==2.24.0 google-cloud-bigquery==3.18.0 -pandas-gbq==0.21.0 +pandas-gbq==0.22.0 pandas===2.0.3; python_version == '3.8' pandas==2.2.1; python_version >= '3.9' -pyarrow==15.0.0; python_version >= '3.8' +pyarrow==15.0.1; python_version >= '3.8' From 7dda6dbaaec03bbd4a66f07bcb6c8bba8fb53b92 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Wed, 13 Mar 2024 19:03:50 +0100 Subject: [PATCH 397/519] chore(deps): update all dependencies (#743) --- packages/pandas-gbq/samples/snippets/requirements-test.txt | 2 +- packages/pandas-gbq/samples/snippets/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements-test.txt b/packages/pandas-gbq/samples/snippets/requirements-test.txt index 271c772d8da7..69e34f186a46 100644 --- a/packages/pandas-gbq/samples/snippets/requirements-test.txt +++ b/packages/pandas-gbq/samples/snippets/requirements-test.txt @@ -1,2 +1,2 @@ google-cloud-testutils==1.4.0 -pytest==8.0.2 +pytest==8.1.1 diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index f632e34569ec..0cafbfbe9606 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,5 +1,5 @@ google-cloud-bigquery-storage==2.24.0 -google-cloud-bigquery==3.18.0 +google-cloud-bigquery==3.19.0 pandas-gbq==0.22.0 pandas===2.0.3; python_version == '3.8' pandas==2.2.1; python_version >= '3.9' From 3112ab008fa89cb8e5d6d0bedf821d1868bc6ab4 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Fri, 22 Mar 2024 09:47:02 +0100 Subject: [PATCH 398/519] chore(deps): update dependency pyarrow to v15.0.2 (#747) --- packages/pandas-gbq/samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 0cafbfbe9606..438fe27172e1 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -3,4 +3,4 @@ google-cloud-bigquery==3.19.0 pandas-gbq==0.22.0 pandas===2.0.3; python_version == '3.8' pandas==2.2.1; python_version >= '3.9' -pyarrow==15.0.1; python_version >= '3.8' +pyarrow==15.0.2; python_version >= '3.8' From cf95cdf1cd6015913a82711fb893a1c020143eb9 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 22 Mar 2024 12:41:35 -0500 Subject: [PATCH 399/519] chore(python): update dependencies in /.kokoro (#746) * chore(python): update dependencies in /.kokoro Source-Link: https://github.com/googleapis/synthtool/commit/db94845da69ccdfefd7ce55c84e6cfa74829747e Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:a8a80fc6456e433df53fc2a0d72ca0345db0ddefb409f1b75b118dfd1babd952 * exclude .kokoro/build.sh from templates; restore nox install command in .kokoro/build.sh * formatting --------- Co-authored-by: Owl Bot Co-authored-by: Anthonios Partheniou --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 +- packages/pandas-gbq/.kokoro/build.sh | 4 - .../pandas-gbq/.kokoro/docker/docs/Dockerfile | 4 + .../.kokoro/docker/docs/requirements.in | 1 + .../.kokoro/docker/docs/requirements.txt | 38 ++++++ packages/pandas-gbq/.kokoro/requirements.in | 3 +- packages/pandas-gbq/.kokoro/requirements.txt | 114 ++++++++---------- packages/pandas-gbq/owlbot.py | 34 +----- 8 files changed, 102 insertions(+), 100 deletions(-) create mode 100644 packages/pandas-gbq/.kokoro/docker/docs/requirements.in create mode 100644 packages/pandas-gbq/.kokoro/docker/docs/requirements.txt diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index e4e943e0259a..4bdeef3904e2 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:98f3afd11308259de6e828e37376d18867fd321aba07826e29e4f8d9cab56bad -# created: 2024-02-27T15:56:18.442440378Z + digest: sha256:a8a80fc6456e433df53fc2a0d72ca0345db0ddefb409f1b75b118dfd1babd952 +# created: 2024-03-15T16:25:47.905264637Z diff --git a/packages/pandas-gbq/.kokoro/build.sh b/packages/pandas-gbq/.kokoro/build.sh index 9abf1e992257..e490fe53f95a 100755 --- a/packages/pandas-gbq/.kokoro/build.sh +++ b/packages/pandas-gbq/.kokoro/build.sh @@ -42,12 +42,8 @@ export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json # Setup project id. export PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json") -# Remove old nox -python3 -m pip uninstall --yes --quiet nox-automation - # Install nox python3 -m pip install --upgrade --quiet nox -python3 -m nox --version # If this is a continuous build, send the test log to the FlakyBot. # See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot. diff --git a/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile b/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile index 8e39a2cc438d..bdaf39fe22d0 100644 --- a/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile +++ b/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile @@ -80,4 +80,8 @@ RUN wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \ # Test pip RUN python3 -m pip +# Install build requirements +COPY requirements.txt /requirements.txt +RUN python3 -m pip install --require-hashes -r requirements.txt + CMD ["python3.8"] diff --git a/packages/pandas-gbq/.kokoro/docker/docs/requirements.in b/packages/pandas-gbq/.kokoro/docker/docs/requirements.in new file mode 100644 index 000000000000..816817c672a1 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/docker/docs/requirements.in @@ -0,0 +1 @@ +nox diff --git a/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt b/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt new file mode 100644 index 000000000000..0e5d70f20f83 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt @@ -0,0 +1,38 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --allow-unsafe --generate-hashes requirements.in +# +argcomplete==3.2.3 \ + --hash=sha256:bf7900329262e481be5a15f56f19736b376df6f82ed27576fa893652c5de6c23 \ + --hash=sha256:c12355e0494c76a2a7b73e3a59b09024ca0ba1e279fb9ed6c1b82d5b74b6a70c + # via nox +colorlog==6.8.2 \ + --hash=sha256:3e3e079a41feb5a1b64f978b5ea4f46040a94f11f0e8bbb8261e3dbbeca64d44 \ + --hash=sha256:4dcbb62368e2800cb3c5abd348da7e53f6c362dda502ec27c560b2e58a66bd33 + # via nox +distlib==0.3.8 \ + --hash=sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784 \ + --hash=sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64 + # via virtualenv +filelock==3.13.1 \ + --hash=sha256:521f5f56c50f8426f5e03ad3b281b490a87ef15bc6c526f168290f0c7148d44e \ + --hash=sha256:57dbda9b35157b05fb3e58ee91448612eb674172fab98ee235ccb0b5bee19a1c + # via virtualenv +nox==2024.3.2 \ + --hash=sha256:e53514173ac0b98dd47585096a55572fe504fecede58ced708979184d05440be \ + --hash=sha256:f521ae08a15adbf5e11f16cb34e8d0e6ea521e0b92868f684e91677deb974553 + # via -r requirements.in +packaging==24.0 \ + --hash=sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5 \ + --hash=sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9 + # via nox +platformdirs==4.2.0 \ + --hash=sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068 \ + --hash=sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768 + # via virtualenv +virtualenv==20.25.1 \ + --hash=sha256:961c026ac520bac5f69acb8ea063e8a4f071bcc9457b9c1f28f6b085c511583a \ + --hash=sha256:e08e13ecdca7a0bd53798f356d5831434afa5b07b93f0abdf0797b7a06ffe197 + # via nox diff --git a/packages/pandas-gbq/.kokoro/requirements.in b/packages/pandas-gbq/.kokoro/requirements.in index ec867d9fd65a..fff4d9ce0d0a 100644 --- a/packages/pandas-gbq/.kokoro/requirements.in +++ b/packages/pandas-gbq/.kokoro/requirements.in @@ -1,5 +1,5 @@ gcp-docuploader -gcp-releasetool>=1.10.5 # required for compatibility with cryptography>=39.x +gcp-releasetool>=2 # required for compatibility with cryptography>=42.x importlib-metadata typing-extensions twine @@ -8,3 +8,4 @@ setuptools nox>=2022.11.21 # required to remove dependency on py charset-normalizer<3 click<8.1.0 +cryptography>=42.0.5 diff --git a/packages/pandas-gbq/.kokoro/requirements.txt b/packages/pandas-gbq/.kokoro/requirements.txt index bda8e38c4f31..dd61f5f32018 100644 --- a/packages/pandas-gbq/.kokoro/requirements.txt +++ b/packages/pandas-gbq/.kokoro/requirements.txt @@ -93,40 +93,41 @@ colorlog==6.7.0 \ # via # gcp-docuploader # nox -cryptography==42.0.4 \ - --hash=sha256:01911714117642a3f1792c7f376db572aadadbafcd8d75bb527166009c9f1d1b \ - --hash=sha256:0e89f7b84f421c56e7ff69f11c441ebda73b8a8e6488d322ef71746224c20fce \ - --hash=sha256:12d341bd42cdb7d4937b0cabbdf2a94f949413ac4504904d0cdbdce4a22cbf88 \ - --hash=sha256:15a1fb843c48b4a604663fa30af60818cd28f895572386e5f9b8a665874c26e7 \ - --hash=sha256:1cdcdbd117681c88d717437ada72bdd5be9de117f96e3f4d50dab3f59fd9ab20 \ - --hash=sha256:1df6fcbf60560d2113b5ed90f072dc0b108d64750d4cbd46a21ec882c7aefce9 \ - --hash=sha256:3c6048f217533d89f2f8f4f0fe3044bf0b2090453b7b73d0b77db47b80af8dff \ - --hash=sha256:3e970a2119507d0b104f0a8e281521ad28fc26f2820687b3436b8c9a5fcf20d1 \ - --hash=sha256:44a64043f743485925d3bcac548d05df0f9bb445c5fcca6681889c7c3ab12764 \ - --hash=sha256:4e36685cb634af55e0677d435d425043967ac2f3790ec652b2b88ad03b85c27b \ - --hash=sha256:5f8907fcf57392cd917892ae83708761c6ff3c37a8e835d7246ff0ad251d9298 \ - --hash=sha256:69b22ab6506a3fe483d67d1ed878e1602bdd5912a134e6202c1ec672233241c1 \ - --hash=sha256:6bfadd884e7280df24d26f2186e4e07556a05d37393b0f220a840b083dc6a824 \ - --hash=sha256:6d0fbe73728c44ca3a241eff9aefe6496ab2656d6e7a4ea2459865f2e8613257 \ - --hash=sha256:6ffb03d419edcab93b4b19c22ee80c007fb2d708429cecebf1dd3258956a563a \ - --hash=sha256:810bcf151caefc03e51a3d61e53335cd5c7316c0a105cc695f0959f2c638b129 \ - --hash=sha256:831a4b37accef30cccd34fcb916a5d7b5be3cbbe27268a02832c3e450aea39cb \ - --hash=sha256:887623fe0d70f48ab3f5e4dbf234986b1329a64c066d719432d0698522749929 \ - --hash=sha256:a0298bdc6e98ca21382afe914c642620370ce0470a01e1bef6dd9b5354c36854 \ - --hash=sha256:a1327f280c824ff7885bdeef8578f74690e9079267c1c8bd7dc5cc5aa065ae52 \ - --hash=sha256:c1f25b252d2c87088abc8bbc4f1ecbf7c919e05508a7e8628e6875c40bc70923 \ - --hash=sha256:c3a5cbc620e1e17009f30dd34cb0d85c987afd21c41a74352d1719be33380885 \ - --hash=sha256:ce8613beaffc7c14f091497346ef117c1798c202b01153a8cc7b8e2ebaaf41c0 \ - --hash=sha256:d2a27aca5597c8a71abbe10209184e1a8e91c1fd470b5070a2ea60cafec35bcd \ - --hash=sha256:dad9c385ba8ee025bb0d856714f71d7840020fe176ae0229de618f14dae7a6e2 \ - --hash=sha256:db4b65b02f59035037fde0998974d84244a64c3265bdef32a827ab9b63d61b18 \ - --hash=sha256:e09469a2cec88fb7b078e16d4adec594414397e8879a4341c6ace96013463d5b \ - --hash=sha256:e53dc41cda40b248ebc40b83b31516487f7db95ab8ceac1f042626bc43a2f992 \ - --hash=sha256:f1e85a178384bf19e36779d91ff35c7617c885da487d689b05c1366f9933ad74 \ - --hash=sha256:f47be41843200f7faec0683ad751e5ef11b9a56a220d57f300376cd8aba81660 \ - --hash=sha256:fb0cef872d8193e487fc6bdb08559c3aa41b659a7d9be48b2e10747f47863925 \ - --hash=sha256:ffc73996c4fca3d2b6c1c8c12bfd3ad00def8621da24f547626bf06441400449 +cryptography==42.0.5 \ + --hash=sha256:0270572b8bd2c833c3981724b8ee9747b3ec96f699a9665470018594301439ee \ + --hash=sha256:111a0d8553afcf8eb02a4fea6ca4f59d48ddb34497aa8706a6cf536f1a5ec576 \ + --hash=sha256:16a48c23a62a2f4a285699dba2e4ff2d1cff3115b9df052cdd976a18856d8e3d \ + --hash=sha256:1b95b98b0d2af784078fa69f637135e3c317091b615cd0905f8b8a087e86fa30 \ + --hash=sha256:1f71c10d1e88467126f0efd484bd44bca5e14c664ec2ede64c32f20875c0d413 \ + --hash=sha256:2424ff4c4ac7f6b8177b53c17ed5d8fa74ae5955656867f5a8affaca36a27abb \ + --hash=sha256:2bce03af1ce5a5567ab89bd90d11e7bbdff56b8af3acbbec1faded8f44cb06da \ + --hash=sha256:329906dcc7b20ff3cad13c069a78124ed8247adcac44b10bea1130e36caae0b4 \ + --hash=sha256:37dd623507659e08be98eec89323469e8c7b4c1407c85112634ae3dbdb926fdd \ + --hash=sha256:3eaafe47ec0d0ffcc9349e1708be2aaea4c6dd4978d76bf6eb0cb2c13636c6fc \ + --hash=sha256:5e6275c09d2badf57aea3afa80d975444f4be8d3bc58f7f80d2a484c6f9485c8 \ + --hash=sha256:6fe07eec95dfd477eb9530aef5bead34fec819b3aaf6c5bd6d20565da607bfe1 \ + --hash=sha256:7367d7b2eca6513681127ebad53b2582911d1736dc2ffc19f2c3ae49997496bc \ + --hash=sha256:7cde5f38e614f55e28d831754e8a3bacf9ace5d1566235e39d91b35502d6936e \ + --hash=sha256:9481ffe3cf013b71b2428b905c4f7a9a4f76ec03065b05ff499bb5682a8d9ad8 \ + --hash=sha256:98d8dc6d012b82287f2c3d26ce1d2dd130ec200c8679b6213b3c73c08b2b7940 \ + --hash=sha256:a011a644f6d7d03736214d38832e030d8268bcff4a41f728e6030325fea3e400 \ + --hash=sha256:a2913c5375154b6ef2e91c10b5720ea6e21007412f6437504ffea2109b5a33d7 \ + --hash=sha256:a30596bae9403a342c978fb47d9b0ee277699fa53bbafad14706af51fe543d16 \ + --hash=sha256:b03c2ae5d2f0fc05f9a2c0c997e1bc18c8229f392234e8a0194f202169ccd278 \ + --hash=sha256:b6cd2203306b63e41acdf39aa93b86fb566049aeb6dc489b70e34bcd07adca74 \ + --hash=sha256:b7ffe927ee6531c78f81aa17e684e2ff617daeba7f189f911065b2ea2d526dec \ + --hash=sha256:b8cac287fafc4ad485b8a9b67d0ee80c66bf3574f655d3b97ef2e1082360faf1 \ + --hash=sha256:ba334e6e4b1d92442b75ddacc615c5476d4ad55cc29b15d590cc6b86efa487e2 \ + --hash=sha256:ba3e4a42397c25b7ff88cdec6e2a16c2be18720f317506ee25210f6d31925f9c \ + --hash=sha256:c41fb5e6a5fe9ebcd58ca3abfeb51dffb5d83d6775405305bfa8715b76521922 \ + --hash=sha256:cd2030f6650c089aeb304cf093f3244d34745ce0cfcc39f20c6fbfe030102e2a \ + --hash=sha256:cd65d75953847815962c84a4654a84850b2bb4aed3f26fadcc1c13892e1e29f6 \ + --hash=sha256:e4985a790f921508f36f81831817cbc03b102d643b5fcb81cd33df3fa291a1a1 \ + --hash=sha256:e807b3188f9eb0eaa7bbb579b462c5ace579f1cedb28107ce8b48a9f7ad3679e \ + --hash=sha256:f12764b8fffc7a123f641d7d049d382b73f96a34117e0b637b80643169cec8ac \ + --hash=sha256:f8837fe1d6ac4a8052a9a8ddab256bc006242696f03368a4009be7ee3075cdb7 # via + # -r requirements.in # gcp-releasetool # secretstorage distlib==0.3.7 \ @@ -145,9 +146,9 @@ gcp-docuploader==0.6.5 \ --hash=sha256:30221d4ac3e5a2b9c69aa52fdbef68cc3f27d0e6d0d90e220fc024584b8d2318 \ --hash=sha256:b7458ef93f605b9d46a4bf3a8dc1755dad1f31d030c8679edf304e343b347eea # via -r requirements.in -gcp-releasetool==1.16.0 \ - --hash=sha256:27bf19d2e87aaa884096ff941aa3c592c482be3d6a2bfe6f06afafa6af2353e3 \ - --hash=sha256:a316b197a543fd036209d0caba7a8eb4d236d8e65381c80cbc6d7efaa7606d63 +gcp-releasetool==2.0.0 \ + --hash=sha256:3d73480b50ba243f22d7c7ec08b115a30e1c7817c4899781840c26f9c55b8277 \ + --hash=sha256:7aa9fd935ec61e581eb8458ad00823786d91756c25e492f372b2b30962f3c28f # via -r requirements.in google-api-core==2.12.0 \ --hash=sha256:c22e01b1e3c4dcd90998494879612c38d0a3411d1f7b679eb89e2abe3ce1f553 \ @@ -392,29 +393,18 @@ platformdirs==3.11.0 \ --hash=sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3 \ --hash=sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e # via virtualenv -protobuf==3.20.3 \ - --hash=sha256:03038ac1cfbc41aa21f6afcbcd357281d7521b4157926f30ebecc8d4ea59dcb7 \ - --hash=sha256:28545383d61f55b57cf4df63eebd9827754fd2dc25f80c5253f9184235db242c \ - --hash=sha256:2e3427429c9cffebf259491be0af70189607f365c2f41c7c3764af6f337105f2 \ - --hash=sha256:398a9e0c3eaceb34ec1aee71894ca3299605fa8e761544934378bbc6c97de23b \ - --hash=sha256:44246bab5dd4b7fbd3c0c80b6f16686808fab0e4aca819ade6e8d294a29c7050 \ - --hash=sha256:447d43819997825d4e71bf5769d869b968ce96848b6479397e29fc24c4a5dfe9 \ - --hash=sha256:67a3598f0a2dcbc58d02dd1928544e7d88f764b47d4a286202913f0b2801c2e7 \ - --hash=sha256:74480f79a023f90dc6e18febbf7b8bac7508420f2006fabd512013c0c238f454 \ - --hash=sha256:819559cafa1a373b7096a482b504ae8a857c89593cf3a25af743ac9ecbd23480 \ - --hash=sha256:899dc660cd599d7352d6f10d83c95df430a38b410c1b66b407a6b29265d66469 \ - --hash=sha256:8c0c984a1b8fef4086329ff8dd19ac77576b384079247c770f29cc8ce3afa06c \ - --hash=sha256:9aae4406ea63d825636cc11ffb34ad3379335803216ee3a856787bcf5ccc751e \ - --hash=sha256:a7ca6d488aa8ff7f329d4c545b2dbad8ac31464f1d8b1c87ad1346717731e4db \ - --hash=sha256:b6cc7ba72a8850621bfec987cb72623e703b7fe2b9127a161ce61e61558ad905 \ - --hash=sha256:bf01b5720be110540be4286e791db73f84a2b721072a3711efff6c324cdf074b \ - --hash=sha256:c02ce36ec760252242a33967d51c289fd0e1c0e6e5cc9397e2279177716add86 \ - --hash=sha256:d9e4432ff660d67d775c66ac42a67cf2453c27cb4d738fc22cb53b5d84c135d4 \ - --hash=sha256:daa564862dd0d39c00f8086f88700fdbe8bc717e993a21e90711acfed02f2402 \ - --hash=sha256:de78575669dddf6099a8a0f46a27e82a1783c557ccc38ee620ed8cc96d3be7d7 \ - --hash=sha256:e64857f395505ebf3d2569935506ae0dfc4a15cb80dc25261176c784662cdcc4 \ - --hash=sha256:f4bd856d702e5b0d96a00ec6b307b0f51c1982c2bf9c0052cf9019e9a544ba99 \ - --hash=sha256:f4c42102bc82a51108e449cbb32b19b180022941c727bac0cfd50170341f16ee +protobuf==4.25.3 \ + --hash=sha256:19b270aeaa0099f16d3ca02628546b8baefe2955bbe23224aaf856134eccf1e4 \ + --hash=sha256:209ba4cc916bab46f64e56b85b090607a676f66b473e6b762e6f1d9d591eb2e8 \ + --hash=sha256:25b5d0b42fd000320bd7830b349e3b696435f3b329810427a6bcce6a5492cc5c \ + --hash=sha256:7c8daa26095f82482307bc717364e7c13f4f1c99659be82890dcfc215194554d \ + --hash=sha256:c053062984e61144385022e53678fbded7aea14ebb3e0305ae3592fb219ccfa4 \ + --hash=sha256:d4198877797a83cbfe9bffa3803602bbe1625dc30d8a097365dbc762e5790faa \ + --hash=sha256:e3c97a1555fd6388f857770ff8b9703083de6bf1f9274a002a332d65fbb56c8c \ + --hash=sha256:e7cb0ae90dd83727f0c0718634ed56837bfeeee29a5f82a7514c03ee1364c019 \ + --hash=sha256:f0700d54bcf45424477e46a9f0944155b46fb0639d69728739c0e47bab83f2b9 \ + --hash=sha256:f1279ab38ecbfae7e456a108c5c0681e4956d5b1090027c1de0f934dfdb4b35c \ + --hash=sha256:f4f118245c4a087776e0a8408be33cf09f6c547442c00395fbfb116fac2f8ac2 # via # gcp-docuploader # gcp-releasetool @@ -518,7 +508,7 @@ zipp==3.17.0 \ # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -setuptools==68.2.2 \ - --hash=sha256:4ac1475276d2f1c48684874089fefcd83bd7162ddaafb81fac866ba0db282a87 \ - --hash=sha256:b454a35605876da60632df1a60f736524eb73cc47bbc9f3f1ef1b644de74fd2a +setuptools==69.2.0 \ + --hash=sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e \ + --hash=sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c # via -r requirements.in diff --git a/packages/pandas-gbq/owlbot.py b/packages/pandas-gbq/owlbot.py index fffbb827e456..ec75c144a929 100644 --- a/packages/pandas-gbq/owlbot.py +++ b/packages/pandas-gbq/owlbot.py @@ -15,7 +15,6 @@ """This script is used to synthesize generated parts of this library.""" import pathlib -import re import synthtool as s from synthtool import gcp @@ -56,7 +55,9 @@ # creating clients, not the end user. "docs/multiprocessing.rst", "noxfile.py", - "README.rst", + "README.rst", + # exclude .kokoro/build.sh which is customized due to support for conda + ".kokoro/build.sh", ], ) @@ -68,35 +69,6 @@ [".github/header-checker-lint.yml"], '"Google LLC"', '"pandas-gbq Authors"', ) -# Work around bug in templates https://github.com/googleapis/synthtool/pull/1335 -s.replace(".github/workflows/unittest.yml", "--fail-under=100", "--fail-under=96") - -# Add environment variables to build.sh to support conda virtualenv -# installations -s.replace( - [".kokoro/build.sh"], - "export PYTHONUNBUFFERED=1", - r"""export PYTHONUNBUFFERED=1 -export CONDA_EXE=/root/conda/bin/conda -export CONDA_PREFIX=/root/conda -export CONDA_PROMPT_MODIFIER=(base) -export _CE_CONDA= -export CONDA_SHLVL=1 -export CONDA_PYTHON_EXE=/root/conda/bin/python -export CONDA_DEFAULT_ENV=base -export PATH=/root/conda/bin:/root/conda/condabin:${PATH} -""", -) - - -# Enable display of all environmental variables, not just KOKORO related vars -s.replace( - [".kokoro/build.sh"], - r"env \| grep KOKORO", - "env", -) - - # ---------------------------------------------------------------------------- # Samples templates # ---------------------------------------------------------------------------- From 28dd8e4ac8d5174c743ccc9fccac652f7bfd58ad Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Fri, 5 Apr 2024 20:14:09 +0200 Subject: [PATCH 400/519] chore(deps): update all dependencies (#750) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update all dependencies * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot --- packages/pandas-gbq/samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 438fe27172e1..5122bd8f21a2 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,5 +1,5 @@ google-cloud-bigquery-storage==2.24.0 -google-cloud-bigquery==3.19.0 +google-cloud-bigquery==3.20.1 pandas-gbq==0.22.0 pandas===2.0.3; python_version == '3.8' pandas==2.2.1; python_version >= '3.9' From 469837421ab52782b75d7261cbc340066e409035 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 14:09:45 -0700 Subject: [PATCH 401/519] docs: add summary_overview template (#751) Source-Link: https://github.com/googleapis/synthtool/commit/d7c2271d319aeb7e3043ec3f1ecec6f3604f1f1e Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:db05f70829de86fe8e34ba972b7fe56da57eaccf1691f875ed4867db80d5cec9 Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 ++-- packages/pandas-gbq/.github/auto-label.yaml | 5 +++++ packages/pandas-gbq/.github/blunderbuss.yml | 17 +++++++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 packages/pandas-gbq/.github/blunderbuss.yml diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 4bdeef3904e2..3189719173b1 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:a8a80fc6456e433df53fc2a0d72ca0345db0ddefb409f1b75b118dfd1babd952 -# created: 2024-03-15T16:25:47.905264637Z + digest: sha256:db05f70829de86fe8e34ba972b7fe56da57eaccf1691f875ed4867db80d5cec9 +# created: 2024-04-05T19:51:26.466869535Z diff --git a/packages/pandas-gbq/.github/auto-label.yaml b/packages/pandas-gbq/.github/auto-label.yaml index b2016d119b40..8b37ee89711f 100644 --- a/packages/pandas-gbq/.github/auto-label.yaml +++ b/packages/pandas-gbq/.github/auto-label.yaml @@ -13,3 +13,8 @@ # limitations under the License. requestsize: enabled: true + +path: + pullrequest: true + paths: + samples: "samples" diff --git a/packages/pandas-gbq/.github/blunderbuss.yml b/packages/pandas-gbq/.github/blunderbuss.yml new file mode 100644 index 000000000000..5b7383dc7665 --- /dev/null +++ b/packages/pandas-gbq/.github/blunderbuss.yml @@ -0,0 +1,17 @@ +# Blunderbuss config +# +# This file controls who is assigned for pull requests and issues. +# Note: This file is autogenerated. To make changes to the assignee +# team, please update `codeowner_team` in `.repo-metadata.json`. +assign_issues: + - googleapis/api-bigquery + +assign_issues_by: + - labels: + - "samples" + to: + - googleapis/python-samples-reviewers + - googleapis/api-bigquery + +assign_prs: + - googleapis/api-bigquery From 4869717aeb2755b86d3de2ecb3c5280118edbc48 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 14:48:10 -0400 Subject: [PATCH 402/519] chore(python): bump idna from 3.4 to 3.7 in .kokoro (#757) Source-Link: https://github.com/googleapis/synthtool/commit/d50980e704793a2d3310bfb3664f3a82f24b5796 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:5a4c19d17e597b92d786e569be101e636c9c2817731f80a5adec56b2aa8fe070 Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 ++-- packages/pandas-gbq/.kokoro/requirements.txt | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 3189719173b1..81f87c56917d 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:db05f70829de86fe8e34ba972b7fe56da57eaccf1691f875ed4867db80d5cec9 -# created: 2024-04-05T19:51:26.466869535Z + digest: sha256:5a4c19d17e597b92d786e569be101e636c9c2817731f80a5adec56b2aa8fe070 +# created: 2024-04-12T11:35:58.922854369Z diff --git a/packages/pandas-gbq/.kokoro/requirements.txt b/packages/pandas-gbq/.kokoro/requirements.txt index dd61f5f32018..51f92b8e12f1 100644 --- a/packages/pandas-gbq/.kokoro/requirements.txt +++ b/packages/pandas-gbq/.kokoro/requirements.txt @@ -252,9 +252,9 @@ googleapis-common-protos==1.61.0 \ --hash=sha256:22f1915393bb3245343f6efe87f6fe868532efc12aa26b391b15132e1279f1c0 \ --hash=sha256:8a64866a97f6304a7179873a465d6eee97b7a24ec6cfd78e0f575e96b821240b # via google-api-core -idna==3.4 \ - --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 \ - --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2 +idna==3.7 \ + --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \ + --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0 # via requests importlib-metadata==6.8.0 \ --hash=sha256:3ebb78df84a805d7698245025b975d9d67053cd94c79245ba4b3eb694abe68bb \ From d86dd484975788b6c1baac0bdf454eec5a807ede Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Fri, 12 Apr 2024 20:48:22 +0200 Subject: [PATCH 403/519] chore(deps): update all dependencies (#755) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update all dependencies * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot --- packages/pandas-gbq/samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 5122bd8f21a2..f6e094935823 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -2,5 +2,5 @@ google-cloud-bigquery-storage==2.24.0 google-cloud-bigquery==3.20.1 pandas-gbq==0.22.0 pandas===2.0.3; python_version == '3.8' -pandas==2.2.1; python_version >= '3.9' +pandas==2.2.2; python_version >= '3.9' pyarrow==15.0.2; python_version >= '3.8' From 29d164ae466578f46d909986d1ca0694fb4800cb Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Fri, 3 May 2024 10:47:08 -0400 Subject: [PATCH 404/519] test: add `pip freeze` to several sessions in the noxfile to aid in troubleshooting (#764) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add pip freeze to various noxfile tests * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot --- packages/pandas-gbq/noxfile.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 6ee180ef4871..49ec38f68b8f 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -256,6 +256,8 @@ def system(session): install_systemtest_dependencies(session, "-c", constraints_path) + session.run("python", "-m", "pip", "freeze") + # Run py.test against the system tests. if system_test_exists: session.run( @@ -511,6 +513,7 @@ def prerelease_deps(session): "requests", ] session.install(*other_deps) + session.run("python", "-m", "pip", "freeze") # Print out prerelease package versions session.run( From d3f61e2c2f920aae8ea03ee811d88e4a9827eeeb Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Wed, 8 May 2024 05:29:29 -0400 Subject: [PATCH 405/519] bug: add missing latency check (#763) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * specify a particular version of bigquery to debug * again tweaking the versions to debug issue. * add some pip freeze commands for debugging * updates minimum latency to correct a flaky bot issue and protect users * Update noxfile.py * Update noxfile.py * Update setup.py * Update noxfile.py * add several test cases to test validation logic * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot --- packages/pandas-gbq/noxfile.py | 8 +++++- packages/pandas-gbq/pandas_gbq/gbq.py | 15 +++++++++- packages/pandas-gbq/tests/system/test_gbq.py | 30 +++++++++++++++++++- 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 49ec38f68b8f..2b973857cdf1 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -256,6 +256,7 @@ def system(session): install_systemtest_dependencies(session, "-c", constraints_path) + # Print out package versions. session.run("python", "-m", "pip", "freeze") # Run py.test against the system tests. @@ -352,12 +353,15 @@ def prerelease(session): "--quiet", f"--junitxml=prerelease_unit_{session.python}_sponge_log.xml", os.path.join("tests", "unit"), + *session.posargs, ) + session.run( "py.test", "--quiet", f"--junitxml=prerelease_system_{session.python}_sponge_log.xml", os.path.join("tests", "system"), + *session.posargs, ) @@ -515,7 +519,9 @@ def prerelease_deps(session): session.install(*other_deps) session.run("python", "-m", "pip", "freeze") - # Print out prerelease package versions + # Print out package versions. + session.run("python", "-m", "pip", "freeze") + session.run( "python", "-c", "import google.protobuf; print(google.protobuf.__version__)" ) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index f93999a9118d..60bb8bdad056 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -411,7 +411,20 @@ def run_query(self, query, max_results=None, progress_bar_type=None, **kwargs): timeout_ms = job_config_dict.get("jobTimeoutMs") or job_config_dict[ "query" ].get("timeoutMs") - timeout_ms = int(timeout_ms) if timeout_ms else None + + if timeout_ms: + timeout_ms = int(timeout_ms) + # Having too small a timeout_ms results in individual + # API calls timing out before they can finish. + # ~300 milliseconds is rule of thumb for bare minimum + # latency from the BigQuery API. + minimum_latency = 400 + if timeout_ms < minimum_latency: + raise QueryTimeout( + f"Query timeout must be at least 400 milliseconds: timeout_ms equals {timeout_ms}." + ) + else: + timeout_ms = None self._start_timer() job_config = bigquery.QueryJobConfig.from_api_repr(job_config_dict) diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index 7afa4ae93873..9bbefceba40d 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -473,13 +473,41 @@ def test_timeout_configuration(self, project_id): sql_statement = """ select count(*) from unnest(generate_array(1,1000000)), unnest(generate_array(1, 10000)) """ + + # This first test confirms that we get a timeout error if we exceed the timeout limit. + # The above query is expected to take a long time and exceed the limit. configs = [ + # we have a minimum limit on the timeout_ms being 400 milliseconds + # see pandas-gbq/gbq.py/GbqConnector/run_query docstring + # for more details. # pandas-gbq timeout configuration. Transformed to REST API compatible version. - {"query": {"useQueryCache": False, "timeoutMs": 1}}, + {"query": {"useQueryCache": False, "timeoutMs": 401}}, # REST API job timeout. See: # https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfiguration.FIELDS.job_timeout_ms + {"query": {"useQueryCache": False}, "jobTimeoutMs": 401}, + ] + for config in configs: + with pytest.raises(gbq.QueryTimeout): + gbq.read_gbq( + sql_statement, + project_id=project_id, + credentials=self.credentials, + configuration=config, + ) + + # This second test confirms out our validation logic won't allow a + # value less than or equal to 400 be used as a timeout value. + # by exercising the system for various edge cases to ensure we catch + # invalid values less than or equal to 400. + configs = [ + {"query": {"useQueryCache": False, "timeoutMs": 399}}, + {"query": {"useQueryCache": False, "timeoutMs": 400}}, + {"query": {"useQueryCache": False, "timeoutMs": 1}}, + {"query": {"useQueryCache": False}, "jobTimeoutMs": 399}, + {"query": {"useQueryCache": False}, "jobTimeoutMs": 400}, {"query": {"useQueryCache": False}, "jobTimeoutMs": 1}, ] + for config in configs: with pytest.raises(gbq.QueryTimeout): gbq.read_gbq( From 2fa8aff96f7170d7c68c449421fbb3aedebd072a Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Thu, 9 May 2024 23:36:15 +0200 Subject: [PATCH 406/519] chore(deps): update all dependencies (#766) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update all dependencies * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot --- packages/pandas-gbq/samples/snippets/requirements-test.txt | 2 +- packages/pandas-gbq/samples/snippets/requirements.txt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements-test.txt b/packages/pandas-gbq/samples/snippets/requirements-test.txt index 69e34f186a46..de395192cf1f 100644 --- a/packages/pandas-gbq/samples/snippets/requirements-test.txt +++ b/packages/pandas-gbq/samples/snippets/requirements-test.txt @@ -1,2 +1,2 @@ google-cloud-testutils==1.4.0 -pytest==8.1.1 +pytest==8.2.0 diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index f6e094935823..e90201cade65 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,6 +1,6 @@ -google-cloud-bigquery-storage==2.24.0 -google-cloud-bigquery==3.20.1 +google-cloud-bigquery-storage==2.25.0 +google-cloud-bigquery==3.22.0 pandas-gbq==0.22.0 pandas===2.0.3; python_version == '3.8' pandas==2.2.2; python_version >= '3.9' -pyarrow==15.0.2; python_version >= '3.8' +pyarrow==16.0.0; python_version >= '3.8' From c49a21f39cdf379e2ce5fc6ba9f42a88cf42fc16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Swe=C3=B1a=20=28Swast=29?= Date: Mon, 20 May 2024 15:58:14 -0500 Subject: [PATCH 407/519] feat: `read_gbq` suggests using BigQuery DataFrames with large results (#769) * feat: `read_gbq` suggests using BigQuery DataFrames with large results * update docs * guard against non-int bytes * tweak message * remove unnecessary also * remove dead code * remove directory that doesn't exist * comment about GiB vs GB --- packages/pandas-gbq/docs/index.rst | 6 ++ packages/pandas-gbq/noxfile.py | 9 ++ packages/pandas-gbq/pandas_gbq/constants.py | 12 +++ packages/pandas-gbq/pandas_gbq/exceptions.py | 4 + packages/pandas-gbq/pandas_gbq/features.py | 10 -- packages/pandas-gbq/pandas_gbq/gbq.py | 59 ++++++++--- packages/pandas-gbq/tests/unit/test_gbq.py | 99 ++++++++++++++----- packages/pandas-gbq/tests/unit/test_to_gbq.py | 5 +- 8 files changed, 153 insertions(+), 51 deletions(-) create mode 100644 packages/pandas-gbq/pandas_gbq/constants.py diff --git a/packages/pandas-gbq/docs/index.rst b/packages/pandas-gbq/docs/index.rst index 67496f2641a8..73673e0f17a9 100644 --- a/packages/pandas-gbq/docs/index.rst +++ b/packages/pandas-gbq/docs/index.rst @@ -23,6 +23,12 @@ Note: The canonical version of this documentation can always be found on the `BigQuery sandbox `__ to try the service for free. + Also, consider using `BigQuery DataFrames + `__ + to process large results with pandas compatible APIs with transparent SQL + pushdown to BigQuery engine. This provides an opportunity to save on costs + and improve performance. + While BigQuery uses standard SQL syntax, it has some important differences from traditional databases both in functionality, API limitations (size and quantity of queries or uploads), and how Google charges for use of the diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 2b973857cdf1..f7b290f4f103 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -375,6 +375,15 @@ def cover(session): session.install("coverage", "pytest-cov") session.run("coverage", "report", "--show-missing", "--fail-under=96") + # Make sure there is no dead code in our test directories. + session.run( + "coverage", + "report", + "--show-missing", + "--include=tests/unit/*", + "--fail-under=100", + ) + session.run("coverage", "erase") diff --git a/packages/pandas-gbq/pandas_gbq/constants.py b/packages/pandas-gbq/pandas_gbq/constants.py new file mode 100644 index 000000000000..37266b3ce9af --- /dev/null +++ b/packages/pandas-gbq/pandas_gbq/constants.py @@ -0,0 +1,12 @@ +# Copyright (c) 2024 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# BigQuery uses powers of 2 in calculating data sizes. See: +# https://cloud.google.com/bigquery/pricing#data The documentation uses +# GiB rather than GB to disambiguate from the alternative base 10 units. +# https://en.wikipedia.org/wiki/Byte#Multiple-byte_units +BYTES_IN_KIB = 1024 +BYTES_IN_MIB = 1024 * BYTES_IN_KIB +BYTES_IN_GIB = 1024 * BYTES_IN_MIB +BYTES_TO_RECOMMEND_BIGFRAMES = BYTES_IN_GIB diff --git a/packages/pandas-gbq/pandas_gbq/exceptions.py b/packages/pandas-gbq/pandas_gbq/exceptions.py index 574b2dec3555..af58212ee91f 100644 --- a/packages/pandas-gbq/pandas_gbq/exceptions.py +++ b/packages/pandas-gbq/pandas_gbq/exceptions.py @@ -28,6 +28,10 @@ class InvalidPrivateKeyFormat(ValueError): """ +class LargeResultsWarning(UserWarning): + """Raise when results are beyond that recommended for pandas DataFrame.""" + + class PerformanceWarning(RuntimeWarning): """ Raised when a performance-related feature is requested, but unsupported. diff --git a/packages/pandas-gbq/pandas_gbq/features.py b/packages/pandas-gbq/pandas_gbq/features.py index 45a43c55f3f9..b6ab25acba5e 100644 --- a/packages/pandas-gbq/pandas_gbq/features.py +++ b/packages/pandas-gbq/pandas_gbq/features.py @@ -9,7 +9,6 @@ BIGQUERY_QUERY_AND_WAIT_VERSION = "3.14.0" PANDAS_VERBOSITY_DEPRECATION_VERSION = "0.23.0" PANDAS_BOOLEAN_DTYPE_VERSION = "1.0.0" -PANDAS_PARQUET_LOSSLESS_TIMESTAMP_VERSION = "1.1.0" class Features: @@ -82,14 +81,5 @@ def pandas_has_boolean_dtype(self): desired_version = packaging.version.parse(PANDAS_BOOLEAN_DTYPE_VERSION) return self.pandas_installed_version >= desired_version - @property - def pandas_has_parquet_with_lossless_timestamp(self): - import packaging.version - - desired_version = packaging.version.parse( - PANDAS_PARQUET_LOSSLESS_TIMESTAMP_VERSION - ) - return self.pandas_installed_version >= desired_version - FEATURES = Features() diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 60bb8bdad056..a9dca3ce421c 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -19,6 +19,8 @@ if typing.TYPE_CHECKING: # pragma: NO COVER import pandas +import pandas_gbq.constants +import pandas_gbq.exceptions from pandas_gbq.exceptions import GenericGBQException, QueryTimeout from pandas_gbq.features import FEATURES import pandas_gbq.query @@ -478,6 +480,35 @@ def _download_results( if max_results is not None: create_bqstorage_client = False + # If we're downloading a large table, BigQuery DataFrames might be a + # better fit. Not all code paths will populate rows_iter._table, but + # if it's not populated that means we are working with a small result + # set. + if (table_ref := getattr(rows_iter, "_table", None)) is not None: + table = self.client.get_table(table_ref) + if ( + isinstance((num_bytes := table.num_bytes), int) + and num_bytes > pandas_gbq.constants.BYTES_TO_RECOMMEND_BIGFRAMES + ): + num_gib = num_bytes / pandas_gbq.constants.BYTES_IN_GIB + warnings.warn( + f"Recommendation: Your results are {num_gib:.1f} GiB. " + "Consider using BigQuery DataFrames " + "(https://cloud.google.com/bigquery/docs/bigquery-dataframes-introduction) " + "to process large results with pandas compatible APIs with transparent SQL " + "pushdown to BigQuery engine. This provides an opportunity to save on costs " + "and improve performance. " + "Please reach out to bigframes-feedback@google.com with any " + "questions or concerns. To disable this message, run " + "warnings.simplefilter('ignore', category=pandas_gbq.exceptions.LargeResultsWarning)", + category=pandas_gbq.exceptions.LargeResultsWarning, + # user's code + # -> read_gbq + # -> run_query + # -> download_results + stacklevel=4, + ) + try: schema_fields = [field.to_api_repr() for field in rows_iter.schema] conversion_dtypes = _bqschema_to_nullsafe_dtypes(schema_fields) @@ -663,18 +694,25 @@ def read_gbq( *, col_order=None, ): - r"""Load data from Google BigQuery using google-cloud-python - - The main method a user calls to execute a Query in Google BigQuery - and read results into a pandas DataFrame. + r"""Read data from Google BigQuery to a pandas DataFrame. - This method uses the Google Cloud client library to make requests to - Google BigQuery, documented `here - `__. + Run a SQL query in BigQuery or read directly from a table + the `Python client library for BigQuery + `__ + and for `BigQuery Storage + `__ + to make API requests. See the :ref:`How to authenticate with Google BigQuery ` guide for authentication instructions. + .. note:: + Consider using `BigQuery DataFrames + `__ to + process large results with pandas compatible APIs that run in the + BigQuery SQL query engine. This provides an opportunity to save on + costs and improve performance. + Parameters ---------- query_or_table : str @@ -1050,12 +1088,7 @@ def to_gbq( ) if api_method == "default": - # Avoid using parquet if pandas doesn't support lossless conversions to - # parquet timestamp. See: https://stackoverflow.com/a/69758676/101923 - if FEATURES.pandas_has_parquet_with_lossless_timestamp: - api_method = "load_parquet" - else: - api_method = "load_csv" + api_method = "load_parquet" if chunksize is not None: if api_method == "load_parquet": diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index 8ba81b6d5a3e..cef916f2f566 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -6,10 +6,13 @@ import copy import datetime +import re from unittest import mock +import warnings import google.api_core.exceptions import google.cloud.bigquery +import google.cloud.bigquery.table import numpy import packaging.version import pandas @@ -17,6 +20,8 @@ import pytest from pandas_gbq import gbq +import pandas_gbq.constants +import pandas_gbq.exceptions import pandas_gbq.features from pandas_gbq.features import FEATURES @@ -147,6 +152,62 @@ def test__transform_read_gbq_configuration_makes_copy(original, expected): assert did_change == should_change +def test_GbqConnector_download_results_warns_for_large_tables(default_bigquery_client): + gbq._test_google_api_imports() + connector = _make_connector() + rows_iter = mock.create_autospec( + google.cloud.bigquery.table.RowIterator, instance=True + ) + table = google.cloud.bigquery.Table.from_api_repr( + { + "tableReference": { + "projectId": "my-proj", + "datasetId": "my-dset", + "tableId": "my_tbl", + }, + "numBytes": 2 * pandas_gbq.constants.BYTES_IN_GIB, + }, + ) + rows_iter._table = table + default_bigquery_client.get_table.reset_mock(side_effect=True) + default_bigquery_client.get_table.return_value = table + + with pytest.warns( + pandas_gbq.exceptions.LargeResultsWarning, + match=re.escape("Your results are 2.0 GiB. Consider using BigQuery DataFrames"), + ): + connector._download_results(rows_iter) + + +def test_GbqConnector_download_results_doesnt_warn_for_small_tables( + default_bigquery_client, +): + gbq._test_google_api_imports() + connector = _make_connector() + rows_iter = mock.create_autospec( + google.cloud.bigquery.table.RowIterator, instance=True + ) + table = google.cloud.bigquery.Table.from_api_repr( + { + "tableReference": { + "projectId": "my-proj", + "datasetId": "my-dset", + "tableId": "my_tbl", + }, + "numBytes": 999 * pandas_gbq.constants.BYTES_IN_MIB, + }, + ) + rows_iter._table = table + default_bigquery_client.get_table.reset_mock(side_effect=True) + default_bigquery_client.get_table.return_value = table + + with warnings.catch_warnings(): + warnings.simplefilter( + "error", category=pandas_gbq.exceptions.LargeResultsWarning + ) + connector._download_results(rows_iter) + + def test_GbqConnector_get_client_w_new_bq(mock_bigquery_client): gbq._test_google_api_imports() pytest.importorskip("google.api_core.client_info") @@ -191,16 +252,13 @@ def test_to_gbq_with_chunksize_warns_deprecation( api_method, warning_message, warning_type ): with pytest.warns(warning_type, match=warning_message): - try: - gbq.to_gbq( - DataFrame([[1]]), - "dataset.tablename", - project_id="my-project", - api_method=api_method, - chunksize=100, - ) - except gbq.TableCreationError: - pass + gbq.to_gbq( + DataFrame([[1]]), + "dataset.tablename", + project_id="my-project", + api_method=api_method, + chunksize=100, + ) @pytest.mark.parametrize(["verbose"], [(True,), (False,)]) @@ -211,15 +269,12 @@ def test_to_gbq_with_verbose_new_pandas_warns_deprecation(monkeypatch, verbose): mock.PropertyMock(return_value=True), ) with pytest.warns(FutureWarning, match="verbose is deprecated"): - try: - gbq.to_gbq( - DataFrame([[1]]), - "dataset.tablename", - project_id="my-project", - verbose=verbose, - ) - except gbq.TableCreationError: - pass + gbq.to_gbq( + DataFrame([[1]]), + "dataset.tablename", + project_id="my-project", + verbose=verbose, + ) def test_to_gbq_with_private_key_raises_notimplementederror(): @@ -233,11 +288,7 @@ def test_to_gbq_with_private_key_raises_notimplementederror(): def test_to_gbq_doesnt_run_query(mock_bigquery_client): - try: - gbq.to_gbq(DataFrame([[1]]), "dataset.tablename", project_id="my-project") - except gbq.TableCreationError: - pass - + gbq.to_gbq(DataFrame([[1]]), "dataset.tablename", project_id="my-project") mock_bigquery_client.query.assert_not_called() diff --git a/packages/pandas-gbq/tests/unit/test_to_gbq.py b/packages/pandas-gbq/tests/unit/test_to_gbq.py index 4456df0ebf10..15176a1b8aab 100644 --- a/packages/pandas-gbq/tests/unit/test_to_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_to_gbq.py @@ -8,14 +8,11 @@ import pytest from pandas_gbq import gbq -from pandas_gbq.features import FEATURES @pytest.fixture def expected_load_method(mock_bigquery_client): - if FEATURES.pandas_has_parquet_with_lossless_timestamp: - return mock_bigquery_client.load_table_from_dataframe - return mock_bigquery_client.load_table_from_file + return mock_bigquery_client.load_table_from_dataframe def test_to_gbq_create_dataset_with_location(mock_bigquery_client): From d2eca0f2466102abd1754fa3166db3bb8e1b3860 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 20 May 2024 22:02:10 +0000 Subject: [PATCH 408/519] chore(main): release 0.23.0 (#771) :robot: I have created a release *beep* *boop* --- ## [0.23.0](https://togithub.com/googleapis/python-bigquery-pandas/compare/v0.22.0...v0.23.0) (2024-05-20) ### Features * `read_gbq` suggests using BigQuery DataFrames with large results ([#769](https://togithub.com/googleapis/python-bigquery-pandas/issues/769)) ([f937edf](https://togithub.com/googleapis/python-bigquery-pandas/commit/f937edf5db910257a367b4cc20d865a38b440f75)) --- This PR was generated with [Release Please](https://togithub.com/googleapis/release-please). See [documentation](https://togithub.com/googleapis/release-please#release-please). --- packages/pandas-gbq/CHANGELOG.md | 7 +++++++ packages/pandas-gbq/pandas_gbq/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index d115e541691b..8743a7df59ad 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.23.0](https://github.com/googleapis/python-bigquery-pandas/compare/v0.22.0...v0.23.0) (2024-05-20) + + +### Features + +* `read_gbq` suggests using BigQuery DataFrames with large results ([#769](https://github.com/googleapis/python-bigquery-pandas/issues/769)) ([f937edf](https://github.com/googleapis/python-bigquery-pandas/commit/f937edf5db910257a367b4cc20d865a38b440f75)) + ## [0.22.0](https://github.com/googleapis/python-bigquery-pandas/compare/v0.21.0...v0.22.0) (2024-03-05) diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py index 6f21e37d7272..661af4a105ee 100644 --- a/packages/pandas-gbq/pandas_gbq/version.py +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -2,4 +2,4 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -__version__ = "0.22.0" +__version__ = "0.23.0" From 5dc0b39039443001f8c2ab21f381750742773082 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Swe=C3=B1a=20=28Swast=29?= Date: Thu, 23 May 2024 14:51:59 -0500 Subject: [PATCH 409/519] docs: use a short-link to BigQuery DataFrames (#773) --- packages/pandas-gbq/docs/index.rst | 4 ++-- packages/pandas-gbq/pandas_gbq/gbq.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/pandas-gbq/docs/index.rst b/packages/pandas-gbq/docs/index.rst index 73673e0f17a9..14fcde286ee2 100644 --- a/packages/pandas-gbq/docs/index.rst +++ b/packages/pandas-gbq/docs/index.rst @@ -23,8 +23,8 @@ Note: The canonical version of this documentation can always be found on the `BigQuery sandbox `__ to try the service for free. - Also, consider using `BigQuery DataFrames - `__ + Also, consider using BigQuery DataFrames + (`bit.ly/bigframes-intro `__) to process large results with pandas compatible APIs with transparent SQL pushdown to BigQuery engine. This provides an opportunity to save on costs and improve performance. diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index a9dca3ce421c..6ae172f9795b 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -493,8 +493,7 @@ def _download_results( num_gib = num_bytes / pandas_gbq.constants.BYTES_IN_GIB warnings.warn( f"Recommendation: Your results are {num_gib:.1f} GiB. " - "Consider using BigQuery DataFrames " - "(https://cloud.google.com/bigquery/docs/bigquery-dataframes-introduction) " + "Consider using BigQuery DataFrames (https://bit.ly/bigframes-intro)" "to process large results with pandas compatible APIs with transparent SQL " "pushdown to BigQuery engine. This provides an opportunity to save on costs " "and improve performance. " From e6a2d593364ba061c3aa8f7ff94d90c018cf0fa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Swe=C3=B1a=20=28Swast=29?= Date: Fri, 24 May 2024 10:52:17 -0500 Subject: [PATCH 410/519] chore: sort imports (#770) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - [ ] Make sure to open an issue as a [bug/issue](https://togithub.com/googleapis/python-bigquery-pandas/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea - [ ] Ensure the tests and linter pass - [ ] Code coverage does not decrease (if any source code was changed) - [ ] Appropriate docs were updated (if necessary) 🦕 --- packages/pandas-gbq/docs/conf.py | 2 +- packages/pandas-gbq/docs/intro.rst | 4 ++-- packages/pandas-gbq/docs/reading.rst | 4 ++-- packages/pandas-gbq/docs/samples | 1 - packages/pandas-gbq/docs/writing.rst | 2 +- packages/pandas-gbq/noxfile.py | 1 + packages/pandas-gbq/owlbot.py | 10 ++++++---- packages/pandas-gbq/pandas_gbq/__init__.py | 4 ++-- packages/pandas-gbq/pandas_gbq/features.py | 2 +- packages/pandas-gbq/pandas_gbq/gbq.py | 11 ++++------- packages/pandas-gbq/pandas_gbq/load.py | 2 +- packages/pandas-gbq/pandas_gbq/query.py | 1 - packages/pandas-gbq/pandas_gbq/schema.py | 1 - packages/pandas-gbq/samples/snippets/conftest.py | 1 - packages/pandas-gbq/samples/snippets/noxfile.py | 1 - packages/pandas-gbq/samples/snippets/read_gbq_test.py | 3 +-- packages/pandas-gbq/samples/snippets/to_gbq_simple.py | 1 - packages/pandas-gbq/setup.py | 1 - packages/pandas-gbq/tests/system/conftest.py | 3 +-- packages/pandas-gbq/tests/system/test_auth.py | 1 - packages/pandas-gbq/tests/system/test_gbq.py | 8 +++----- packages/pandas-gbq/tests/system/test_read_gbq.py | 3 +-- .../tests/system/test_read_gbq_with_bqstorage.py | 1 - packages/pandas-gbq/tests/system/test_to_gbq.py | 3 +-- packages/pandas-gbq/tests/unit/test_context.py | 3 ++- packages/pandas-gbq/tests/unit/test_gbq.py | 1 - packages/pandas-gbq/tests/unit/test_load.py | 3 +-- packages/pandas-gbq/tests/unit/test_query.py | 2 +- packages/pandas-gbq/tests/unit/test_timestamp.py | 1 - packages/pandas-gbq/tests/unit/test_to_gbq.py | 2 +- 30 files changed, 33 insertions(+), 50 deletions(-) delete mode 120000 packages/pandas-gbq/docs/samples diff --git a/packages/pandas-gbq/docs/conf.py b/packages/pandas-gbq/docs/conf.py index 3a9d090ea804..fe502297abf9 100644 --- a/packages/pandas-gbq/docs/conf.py +++ b/packages/pandas-gbq/docs/conf.py @@ -24,9 +24,9 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys import os import shlex +import sys # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the diff --git a/packages/pandas-gbq/docs/intro.rst b/packages/pandas-gbq/docs/intro.rst index c6774b15ec4c..ab93b4fe2ba9 100644 --- a/packages/pandas-gbq/docs/intro.rst +++ b/packages/pandas-gbq/docs/intro.rst @@ -27,7 +27,7 @@ Reading data from BigQuery Use the :func:`pandas_gbq.read_gbq` function to run a BigQuery query and download the results as a :class:`pandas.DataFrame` object. -.. literalinclude:: samples/snippets/read_gbq_simple.py +.. literalinclude:: ../samples/snippets/read_gbq_simple.py :language: python :dedent: 4 :start-after: [START bigquery_pandas_gbq_read_gbq_simple] @@ -57,7 +57,7 @@ Writing data to BigQuery Use the :func:`pandas_gbq.to_gbq` function to write a :class:`pandas.DataFrame` object to a BigQuery table. -.. literalinclude:: samples/snippets/to_gbq_simple.py +.. literalinclude:: ../samples/snippets/to_gbq_simple.py :language: python :dedent: 4 :start-after: [START bigquery_pandas_gbq_to_gbq_simple] diff --git a/packages/pandas-gbq/docs/reading.rst b/packages/pandas-gbq/docs/reading.rst index 6361280ac5da..bc7b74e150b3 100644 --- a/packages/pandas-gbq/docs/reading.rst +++ b/packages/pandas-gbq/docs/reading.rst @@ -6,7 +6,7 @@ Reading Tables Use the :func:`pandas_gbq.read_gbq` function to run a BigQuery query and download the results as a :class:`pandas.DataFrame` object. -.. literalinclude:: samples/snippets/read_gbq_simple.py +.. literalinclude:: ../samples/snippets/read_gbq_simple.py :language: python :dedent: 4 :start-after: [START bigquery_pandas_gbq_read_gbq_simple] @@ -37,7 +37,7 @@ The ``dialect`` argument can be used to indicate whether to use BigQuery's ``'legacy'`` SQL or BigQuery's ``'standard'`` SQL. The default value is ``'standard'``. -.. literalinclude:: samples/snippets/read_gbq_legacy.py +.. literalinclude:: ../samples/snippets/read_gbq_legacy.py :language: python :dedent: 4 :start-after: [START bigquery_pandas_gbq_read_gbq_legacy] diff --git a/packages/pandas-gbq/docs/samples b/packages/pandas-gbq/docs/samples deleted file mode 120000 index e804737ed3a9..000000000000 --- a/packages/pandas-gbq/docs/samples +++ /dev/null @@ -1 +0,0 @@ -../samples \ No newline at end of file diff --git a/packages/pandas-gbq/docs/writing.rst b/packages/pandas-gbq/docs/writing.rst index 6c1be27220a8..80c06a583cc8 100644 --- a/packages/pandas-gbq/docs/writing.rst +++ b/packages/pandas-gbq/docs/writing.rst @@ -6,7 +6,7 @@ Writing Tables Use the :func:`pandas_gbq.to_gbq` function to write a :class:`pandas.DataFrame` object to a BigQuery table. -.. literalinclude:: samples/snippets/to_gbq_simple.py +.. literalinclude:: ../samples/snippets/to_gbq_simple.py :language: python :dedent: 4 :start-after: [START bigquery_pandas_gbq_to_gbq_simple] diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index f7b290f4f103..2faa9bf9c9cd 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -17,6 +17,7 @@ # Generated by synthtool. DO NOT EDIT! from __future__ import absolute_import + import os import pathlib import re diff --git a/packages/pandas-gbq/owlbot.py b/packages/pandas-gbq/owlbot.py index ec75c144a929..af5348692a4a 100644 --- a/packages/pandas-gbq/owlbot.py +++ b/packages/pandas-gbq/owlbot.py @@ -55,7 +55,7 @@ # creating clients, not the end user. "docs/multiprocessing.rst", "noxfile.py", - "README.rst", + "README.rst", # exclude .kokoro/build.sh which is customized due to support for conda ".kokoro/build.sh", ], @@ -66,7 +66,9 @@ # ---------------------------------------------------------------------------- s.replace( - [".github/header-checker-lint.yml"], '"Google LLC"', '"pandas-gbq Authors"', + [".github/header-checker-lint.yml"], + '"Google LLC"', + '"pandas-gbq Authors"', ) # ---------------------------------------------------------------------------- @@ -79,6 +81,6 @@ # Final cleanup # ---------------------------------------------------------------------------- -s.shell.run(["nox", "-s", "blacken"], hide_output=False) +s.shell.run(["nox", "-s", "format"], hide_output=False) for noxfile in REPO_ROOT.glob("samples/**/noxfile.py"): - s.shell.run(["nox", "-s", "blacken"], cwd=noxfile.parent, hide_output=False) + s.shell.run(["nox", "-s", "format"], cwd=noxfile.parent, hide_output=False) diff --git a/packages/pandas-gbq/pandas_gbq/__init__.py b/packages/pandas-gbq/pandas_gbq/__init__.py index df2b603d6496..6d92bfa2190a 100644 --- a/packages/pandas-gbq/pandas_gbq/__init__.py +++ b/packages/pandas-gbq/pandas_gbq/__init__.py @@ -2,10 +2,10 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -from .gbq import to_gbq, read_gbq, Context, context # noqa - from pandas_gbq import version as pandas_gbq_version +from .gbq import Context, context, read_gbq, to_gbq # noqa + __version__ = pandas_gbq_version.__version__ __all__ = [ diff --git a/packages/pandas-gbq/pandas_gbq/features.py b/packages/pandas-gbq/pandas_gbq/features.py index b6ab25acba5e..2871d5eaa3cb 100644 --- a/packages/pandas-gbq/pandas_gbq/features.py +++ b/packages/pandas-gbq/pandas_gbq/features.py @@ -54,8 +54,8 @@ def bigquery_has_query_and_wait(self): @property def pandas_installed_version(self): - import pandas import packaging.version + import pandas if self._pandas_installed_version is not None: return self._pandas_installed_version diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 6ae172f9795b..36aef79e3ffc 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -268,8 +268,8 @@ def __init__( client_secret=None, ): global context - from google.api_core.exceptions import GoogleAPIError - from google.api_core.exceptions import ClientError + from google.api_core.exceptions import ClientError, GoogleAPIError + from pandas_gbq import auth self.http_error = (ClientError, GoogleAPIError) @@ -1244,8 +1244,7 @@ def __init__( def _table_ref(self, table_id): """Return a BigQuery client library table reference""" - from google.cloud.bigquery import DatasetReference - from google.cloud.bigquery import TableReference + from google.cloud.bigquery import DatasetReference, TableReference return TableReference( DatasetReference(self.project_id, self.dataset_id), table_id @@ -1286,9 +1285,7 @@ def create(self, table_id, schema): Use the generate_bq_schema to generate your table schema from a dataframe. """ - from google.cloud.bigquery import DatasetReference - from google.cloud.bigquery import Table - from google.cloud.bigquery import TableReference + from google.cloud.bigquery import DatasetReference, Table, TableReference if self.exists(table_id): raise TableCreationError("Table {0} already exists".format(table_id)) diff --git a/packages/pandas-gbq/pandas_gbq/load.py b/packages/pandas-gbq/pandas_gbq/load.py index 8243c7f3f04a..cf495e00cff3 100644 --- a/packages/pandas-gbq/pandas_gbq/load.py +++ b/packages/pandas-gbq/pandas_gbq/load.py @@ -9,9 +9,9 @@ from typing import Any, Callable, Dict, List, Optional import db_dtypes +from google.cloud import bigquery import pandas import pyarrow.lib -from google.cloud import bigquery from pandas_gbq import exceptions import pandas_gbq.schema diff --git a/packages/pandas-gbq/pandas_gbq/query.py b/packages/pandas-gbq/pandas_gbq/query.py index 0b7036f7a1b7..83575a9cc3c8 100644 --- a/packages/pandas-gbq/pandas_gbq/query.py +++ b/packages/pandas-gbq/pandas_gbq/query.py @@ -14,7 +14,6 @@ import pandas_gbq.exceptions - logger = logging.getLogger(__name__) diff --git a/packages/pandas-gbq/pandas_gbq/schema.py b/packages/pandas-gbq/pandas_gbq/schema.py index d3357719b574..b60fdedab795 100644 --- a/packages/pandas-gbq/pandas_gbq/schema.py +++ b/packages/pandas-gbq/pandas_gbq/schema.py @@ -6,7 +6,6 @@ import copy - # API may return data types as legacy SQL, so maintain a mapping of aliases # from standard SQL to legacy data types. _TYPE_ALIASES = { diff --git a/packages/pandas-gbq/samples/snippets/conftest.py b/packages/pandas-gbq/samples/snippets/conftest.py index 0d0ae0916618..0216c1da504b 100644 --- a/packages/pandas-gbq/samples/snippets/conftest.py +++ b/packages/pandas-gbq/samples/snippets/conftest.py @@ -6,7 +6,6 @@ import pytest import test_utils.prefixer - prefixer = test_utils.prefixer.Prefixer("python-bigquery-pandas", "samples/snippets") diff --git a/packages/pandas-gbq/samples/snippets/noxfile.py b/packages/pandas-gbq/samples/snippets/noxfile.py index 3b7135946fd5..c36d5f2d81f3 100644 --- a/packages/pandas-gbq/samples/snippets/noxfile.py +++ b/packages/pandas-gbq/samples/snippets/noxfile.py @@ -22,7 +22,6 @@ import nox - # WARNING - WARNING - WARNING - WARNING - WARNING # WARNING - WARNING - WARNING - WARNING - WARNING # DO NOT EDIT THIS FILE EVER! diff --git a/packages/pandas-gbq/samples/snippets/read_gbq_test.py b/packages/pandas-gbq/samples/snippets/read_gbq_test.py index 8f4992d71d58..4fc8cb7b9d1f 100644 --- a/packages/pandas-gbq/samples/snippets/read_gbq_test.py +++ b/packages/pandas-gbq/samples/snippets/read_gbq_test.py @@ -4,8 +4,7 @@ """System tests for read_gbq code samples.""" -from . import read_gbq_legacy -from . import read_gbq_simple +from . import read_gbq_legacy, read_gbq_simple def test_read_gbq_legacy(project_id): diff --git a/packages/pandas-gbq/samples/snippets/to_gbq_simple.py b/packages/pandas-gbq/samples/snippets/to_gbq_simple.py index 37f4cdcd46f2..7330e2e06bf5 100644 --- a/packages/pandas-gbq/samples/snippets/to_gbq_simple.py +++ b/packages/pandas-gbq/samples/snippets/to_gbq_simple.py @@ -14,7 +14,6 @@ def main(project_id, table_id): # TODO: Set project_id to your Google Cloud Platform project ID. # project_id = "my-project" - # TODO: Set table_id to the full destination table ID (including the # dataset ID). # table_id = 'my_dataset.my_table' diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index 6f30eacdeb0b..461cbfdee234 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -10,7 +10,6 @@ import setuptools - # Package metadata. name = "pandas-gbq" diff --git a/packages/pandas-gbq/tests/system/conftest.py b/packages/pandas-gbq/tests/system/conftest.py index 9690446dc762..8c45167fb7af 100644 --- a/packages/pandas-gbq/tests/system/conftest.py +++ b/packages/pandas-gbq/tests/system/conftest.py @@ -2,15 +2,14 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -import os import functools +import os import pathlib from google.cloud import bigquery import pytest import test_utils.prefixer - prefixer = test_utils.prefixer.Prefixer("python-bigquery-pandas", "tests/system") REPO_DIR = pathlib.Path(__file__).parent.parent.parent diff --git a/packages/pandas-gbq/tests/system/test_auth.py b/packages/pandas-gbq/tests/system/test_auth.py index ecedd9732256..564625a92d73 100644 --- a/packages/pandas-gbq/tests/system/test_auth.py +++ b/packages/pandas-gbq/tests/system/test_auth.py @@ -11,7 +11,6 @@ from pandas_gbq import auth - IS_RUNNING_ON_CI = "KOKORO_BUILD_ID" in os.environ diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index 9bbefceba40d..b62f35904e7a 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -5,22 +5,20 @@ # -*- coding: utf-8 -*- import datetime -import packaging.version import sys import numpy as np +import packaging.version import pandas +from pandas import DataFrame import pandas.api.types import pandas.testing as tm -from pandas import DataFrame - -import pytz import pytest +import pytz from pandas_gbq import gbq import pandas_gbq.schema - TABLE_ID = "new_test" PANDAS_VERSION = packaging.version.parse(pandas.__version__) diff --git a/packages/pandas-gbq/tests/system/test_read_gbq.py b/packages/pandas-gbq/tests/system/test_read_gbq.py index fada140b075c..4ae96a362e2d 100644 --- a/packages/pandas-gbq/tests/system/test_read_gbq.py +++ b/packages/pandas-gbq/tests/system/test_read_gbq.py @@ -5,18 +5,17 @@ import collections import datetime import decimal -import packaging.version import random import db_dtypes from google.cloud import bigquery +import packaging.version import pandas import pandas.testing import pytest from pandas_gbq.features import FEATURES - QueryTestCase = collections.namedtuple( "QueryTestCase", ["query", "expected", "use_bqstorage_apis"], diff --git a/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py b/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py index cfb31ea81a6e..70dfecf45dfd 100644 --- a/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py +++ b/packages/pandas-gbq/tests/system/test_read_gbq_with_bqstorage.py @@ -9,7 +9,6 @@ import pytest - pytest.importorskip("google.cloud.bigquery", minversion="1.24.0") diff --git a/packages/pandas-gbq/tests/system/test_to_gbq.py b/packages/pandas-gbq/tests/system/test_to_gbq.py index a03113d7724e..2e7245d5575b 100644 --- a/packages/pandas-gbq/tests/system/test_to_gbq.py +++ b/packages/pandas-gbq/tests/system/test_to_gbq.py @@ -2,9 +2,9 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. +import collections import datetime import decimal -import collections import random import db_dtypes @@ -12,7 +12,6 @@ import pandas.testing import pytest - pytest.importorskip("google.cloud.bigquery", minversion="1.24.0") diff --git a/packages/pandas-gbq/tests/unit/test_context.py b/packages/pandas-gbq/tests/unit/test_context.py index 6b6ce6a0744c..451153022dba 100644 --- a/packages/pandas-gbq/tests/unit/test_context.py +++ b/packages/pandas-gbq/tests/unit/test_context.py @@ -28,9 +28,10 @@ def default_bigquery_client(mock_bigquery_client): @pytest.fixture(autouse=True) def mock_get_credentials(monkeypatch): - from pandas_gbq import auth import google.auth.credentials + from pandas_gbq import auth + mock_credentials = mock.MagicMock(google.auth.credentials.Credentials) mock_get_credentials = mock.Mock() mock_get_credentials.return_value = (mock_credentials, "my-project") diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index cef916f2f566..92a09a3fe6af 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -25,7 +25,6 @@ import pandas_gbq.features from pandas_gbq.features import FEATURES - pytestmark = pytest.mark.filterwarnings("ignore:credentials from Google Cloud SDK") diff --git a/packages/pandas-gbq/tests/unit/test_load.py b/packages/pandas-gbq/tests/unit/test_load.py index b34b13782b5e..5f38d244e931 100644 --- a/packages/pandas-gbq/tests/unit/test_load.py +++ b/packages/pandas-gbq/tests/unit/test_load.py @@ -15,8 +15,7 @@ import pandas.testing import pytest -from pandas_gbq import exceptions -from pandas_gbq import load +from pandas_gbq import exceptions, load def load_method(bqclient, api_method): diff --git a/packages/pandas-gbq/tests/unit/test_query.py b/packages/pandas-gbq/tests/unit/test_query.py index 5b63163496a8..2437fa02d897 100644 --- a/packages/pandas-gbq/tests/unit/test_query.py +++ b/packages/pandas-gbq/tests/unit/test_query.py @@ -4,8 +4,8 @@ from __future__ import annotations -import datetime import concurrent.futures +import datetime from unittest import mock import freezegun diff --git a/packages/pandas-gbq/tests/unit/test_timestamp.py b/packages/pandas-gbq/tests/unit/test_timestamp.py index b35c13074c16..682619fe30eb 100644 --- a/packages/pandas-gbq/tests/unit/test_timestamp.py +++ b/packages/pandas-gbq/tests/unit/test_timestamp.py @@ -6,7 +6,6 @@ import pandas import pandas.testing - import pytest diff --git a/packages/pandas-gbq/tests/unit/test_to_gbq.py b/packages/pandas-gbq/tests/unit/test_to_gbq.py index 15176a1b8aab..23b7c9bd9389 100644 --- a/packages/pandas-gbq/tests/unit/test_to_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_to_gbq.py @@ -2,8 +2,8 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -import google.cloud.bigquery import google.api_core.exceptions +import google.cloud.bigquery from pandas import DataFrame import pytest From 9ec16b34bafbb2c56dce53c3bfe4e9332dbe8a31 Mon Sep 17 00:00:00 2001 From: Lingqing Gan Date: Wed, 29 May 2024 11:32:03 -0700 Subject: [PATCH 411/519] fix: handle None when converting numerics to parquet (#768) * fix: handle None when converting numerics to parquet * lint and fix unit test * add check for pandas.NA --- packages/pandas-gbq/pandas_gbq/load.py | 11 ++++++++++- packages/pandas-gbq/tests/unit/test_load.py | 19 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/pandas_gbq/load.py b/packages/pandas-gbq/pandas_gbq/load.py index cf495e00cff3..45e474b2bd54 100644 --- a/packages/pandas-gbq/pandas_gbq/load.py +++ b/packages/pandas-gbq/pandas_gbq/load.py @@ -99,7 +99,16 @@ def cast_dataframe_for_parquet( errors="ignore", ) elif column_type in {"NUMERIC", "DECIMAL", "BIGNUMERIC", "BIGDECIMAL"}: - cast_column = dataframe[column_name].map(decimal.Decimal) + # decimal.Decimal does not support `None` or `pandas.NA` input, add + # support here. + # https://github.com/googleapis/python-bigquery-pandas/issues/719 + def convert(x): + if pandas.isna(x): # true for `None` and `pandas.NA` + return decimal.Decimal("NaN") + else: + return decimal.Decimal(x) + + cast_column = dataframe[column_name].map(convert) else: cast_column = None diff --git a/packages/pandas-gbq/tests/unit/test_load.py b/packages/pandas-gbq/tests/unit/test_load.py index 5f38d244e931..45c735336180 100644 --- a/packages/pandas-gbq/tests/unit/test_load.py +++ b/packages/pandas-gbq/tests/unit/test_load.py @@ -369,3 +369,22 @@ def test_cast_dataframe_for_parquet_w_null_fields(): schema = {"fields": None} result = load.cast_dataframe_for_parquet(dataframe, schema) pandas.testing.assert_frame_equal(result, expected) + + +# Verifies null numerics are properly handled +# https://github.com/googleapis/python-bigquery-pandas/issues/719 +def test_cast_dataframe_for_parquet_w_null_numerics(): + from decimal import Decimal + + nans = pandas.Series([Decimal("3.14"), Decimal("nan"), None, pandas.NA]) + dataframe = pandas.DataFrame({"A": nans}) + + schema = {"fields": [{"name": "A", "type": "BIGNUMERIC"}]} + result = load.cast_dataframe_for_parquet(dataframe, schema) + + # pandas.testing.assert_frame_equal() doesn't distinguish Decimal("NaN") + # vs. None, verify Decimal("NaN") directly. + # https://github.com/pandas-dev/pandas/issues/18463 + assert result["A"][1].is_nan() + assert result["A"][2].is_nan() + assert result["A"][3].is_nan() From 62986ab2c863b620e050d918ccdc7565f42f6d40 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Wed, 29 May 2024 15:40:40 -0400 Subject: [PATCH 412/519] fix: clean up nox sessions to uniformly displays package versions for debug purposes (#777) * specify a particular version of bigquery to debug * fix: updates noxfile to cleanup debugging code * Update noxfile.py --- packages/pandas-gbq/noxfile.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 2faa9bf9c9cd..a8fb6c472363 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -527,16 +527,10 @@ def prerelease_deps(session): "requests", ] session.install(*other_deps) - session.run("python", "-m", "pip", "freeze") # Print out package versions. session.run("python", "-m", "pip", "freeze") - session.run( - "python", "-c", "import google.protobuf; print(google.protobuf.__version__)" - ) - session.run("python", "-c", "import grpc; print(grpc.__version__)") - session.run("py.test", "tests/unit") system_test_path = os.path.join("tests", "system.py") From aaab72b9e96322215b5739a65499e7a16125d1e8 Mon Sep 17 00:00:00 2001 From: Kira Date: Tue, 4 Jun 2024 10:59:04 -0400 Subject: [PATCH 413/519] chore: pip check on conda forge dependencies (#723) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: pip check on conda forge dependencies * reformatted with black * Update noxfile.py * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Chalmer Lowe Co-authored-by: Owl Bot --- packages/pandas-gbq/noxfile.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index a8fb6c472363..3d0ce24427c0 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -22,6 +22,7 @@ import pathlib import re import shutil +import subprocess import warnings import nox @@ -614,6 +615,12 @@ def conda_test(session): # for troubleshooting purposes. session.run("mamba", "list") + # Using subprocess.run() instead of session.run() because + # session.run() does not correctly handle the pip check command. + subprocess.run( + ["pip", "check"], check=True + ) # Raise an exception if pip check fails + # Tests are limited to unit tests only, at this time. session.run( "py.test", From a3b31860ec369bc941a9b1b3e4f2b5bbfda2c627 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Fri, 7 Jun 2024 13:52:20 -0400 Subject: [PATCH 414/519] fix: set minimum allowable version of sqlite when performing a conda install (#780) Fix: sets minimum allowable version of `sqlite` when performing a `conda` install. Conda installs were killing the install of packages in the presubmit `conda_test-3.12` test. See this [check for context](https://source.cloud.google.com/results/invocations/beabfa58-9228-40fb-a60b-48df1944b159). --- packages/pandas-gbq/noxfile.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 3d0ce24427c0..9581c2518043 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -607,6 +607,7 @@ def conda_test(session): "pydata-google-auth", "tqdm", "protobuf", + "sqlite>3.31.1", # v3.31.1 caused test failures ] install_conda_unittest_dependencies(session, standard_deps, conda_forge_packages) From e80c9a06bfceefeb11e98f506eab5d0f67fd5012 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 10:23:50 -0400 Subject: [PATCH 415/519] chore(main): release 0.23.1 (#774) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- packages/pandas-gbq/CHANGELOG.md | 13 +++++++++++++ packages/pandas-gbq/pandas_gbq/version.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index 8743a7df59ad..418201bfa915 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## [0.23.1](https://github.com/googleapis/python-bigquery-pandas/compare/v0.23.0...v0.23.1) (2024-06-07) + + +### Bug Fixes + +* Handle None when converting numerics to parquet ([#768](https://github.com/googleapis/python-bigquery-pandas/issues/768)) ([53a4683](https://github.com/googleapis/python-bigquery-pandas/commit/53a46833a320963d5c15427f6eb631e0199fb332)) +* Set minimum allowable version of sqlite when performing a conda install ([#780](https://github.com/googleapis/python-bigquery-pandas/issues/780)) ([8a03d44](https://github.com/googleapis/python-bigquery-pandas/commit/8a03d44fbe125ae1202f43b7c6e54c98eca94d4d)) + + +### Documentation + +* Use a short-link to BigQuery DataFrames ([#773](https://github.com/googleapis/python-bigquery-pandas/issues/773)) ([7cd4287](https://github.com/googleapis/python-bigquery-pandas/commit/7cd4287ae70861e4949487db578ab1916d853029)) + ## [0.23.0](https://github.com/googleapis/python-bigquery-pandas/compare/v0.22.0...v0.23.0) (2024-05-20) diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py index 661af4a105ee..2de80e86289c 100644 --- a/packages/pandas-gbq/pandas_gbq/version.py +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -2,4 +2,4 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -__version__ = "0.23.0" +__version__ = "0.23.1" From b498e78540f4327b1fbb6819dd8080da687b71d7 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Mon, 10 Jun 2024 14:16:36 -0400 Subject: [PATCH 416/519] bug: modifies latency due to flakybot issues (#781) --- packages/pandas-gbq/pandas_gbq/gbq.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 36aef79e3ffc..19c42a6b04a3 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -419,11 +419,12 @@ def run_query(self, query, max_results=None, progress_bar_type=None, **kwargs): # Having too small a timeout_ms results in individual # API calls timing out before they can finish. # ~300 milliseconds is rule of thumb for bare minimum - # latency from the BigQuery API. - minimum_latency = 400 + # latency from the BigQuery API, however, 400 milliseconds + # produced too many issues with flakybot failures. + minimum_latency = 500 if timeout_ms < minimum_latency: raise QueryTimeout( - f"Query timeout must be at least 400 milliseconds: timeout_ms equals {timeout_ms}." + f"Query timeout must be at least 500 milliseconds: timeout_ms equals {timeout_ms}." ) else: timeout_ms = None From 758412f86163f1bf62b5e9ea287feb4ec931847e Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Mon, 10 Jun 2024 22:23:51 +0200 Subject: [PATCH 417/519] chore(deps): update all dependencies (#782) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update all dependencies * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot Co-authored-by: Lingqing Gan --- packages/pandas-gbq/samples/snippets/requirements-test.txt | 2 +- packages/pandas-gbq/samples/snippets/requirements.txt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements-test.txt b/packages/pandas-gbq/samples/snippets/requirements-test.txt index de395192cf1f..7af5d0fecadb 100644 --- a/packages/pandas-gbq/samples/snippets/requirements-test.txt +++ b/packages/pandas-gbq/samples/snippets/requirements-test.txt @@ -1,2 +1,2 @@ google-cloud-testutils==1.4.0 -pytest==8.2.0 +pytest==8.2.2 diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index e90201cade65..731d869db1c5 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,6 +1,6 @@ google-cloud-bigquery-storage==2.25.0 -google-cloud-bigquery==3.22.0 -pandas-gbq==0.22.0 +google-cloud-bigquery==3.24.0 +pandas-gbq==0.23.0 pandas===2.0.3; python_version == '3.8' pandas==2.2.2; python_version >= '3.9' -pyarrow==16.0.0; python_version >= '3.8' +pyarrow==16.1.0; python_version >= '3.8' From 6fad79c3a664073360810f2338e09c94e3a3ba43 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Tue, 11 Jun 2024 14:11:49 -0400 Subject: [PATCH 418/519] test: upgrading the ci/cd pipeline to reduce turnover time. (#783) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * adds decorator and prerelease tweaks * removes the duplicative prerelease-deps * remove reference to unused nox session/kokoro check * adds two import statements * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * adds exclusion to owlbot.py * corrects syntax error in owlbot.py file --------- Co-authored-by: Owl Bot --- packages/pandas-gbq/noxfile.py | 124 ++++++++++----------------------- packages/pandas-gbq/owlbot.py | 6 +- 2 files changed, 41 insertions(+), 89 deletions(-) diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 9581c2518043..52590f366f93 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -18,11 +18,13 @@ from __future__ import absolute_import +from functools import wraps import os import pathlib import re import shutil import subprocess +import time import warnings import nox @@ -76,6 +78,27 @@ CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() + +def _calculate_duration(func): + """This decorator prints the execution time for the decorated function.""" + + @wraps(func) + def wrapper(*args, **kwargs): + start = time.monotonic() + result = func(*args, **kwargs) + end = time.monotonic() + total_seconds = round(end - start) + hours = total_seconds // 3600 # Integer division to get hours + remaining_seconds = total_seconds % 3600 # Modulo to find remaining seconds + minutes = remaining_seconds // 60 + seconds = remaining_seconds % 60 + human_time = f"{hours:}:{minutes:0>2}:{seconds:0>2}" + print(f"Session ran in {total_seconds} seconds ({human_time})") + return result + + return wrapper + + # 'docfx' is excluded since it only needs to run in 'docs-presubmit' nox.options.sessions = [ "unit", @@ -92,6 +115,7 @@ @nox.session(python=DEFAULT_PYTHON_VERSION) +@_calculate_duration def lint(session): """Run linters. @@ -108,6 +132,7 @@ def lint(session): @nox.session(python=DEFAULT_PYTHON_VERSION) +@_calculate_duration def blacken(session): """Run black. Format code to uniform standard.""" session.install(BLACK_VERSION) @@ -118,6 +143,7 @@ def blacken(session): @nox.session(python=DEFAULT_PYTHON_VERSION) +@_calculate_duration def format(session): """ Run isort to sort imports. Then run black @@ -138,6 +164,7 @@ def format(session): @nox.session(python=DEFAULT_PYTHON_VERSION) +@_calculate_duration def lint_setup_py(session): """Verify that setup.py is valid (including RST check).""" session.install("docutils", "pygments") @@ -199,6 +226,7 @@ def default(session): @nox.session(python=UNIT_TEST_PYTHON_VERSIONS) +@_calculate_duration def unit(session): """Run the unit test suite.""" default(session) @@ -235,6 +263,7 @@ def install_systemtest_dependencies(session, *constraints): @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) +@_calculate_duration def system(session): """Run the system test suite.""" constraints_path = str( @@ -281,6 +310,7 @@ def system(session): @nox.session(python=DEFAULT_PYTHON_VERSION) +@_calculate_duration def prerelease(session): session.install( "--extra-index-url", @@ -334,7 +364,7 @@ def prerelease(session): constraints_text = constraints_file.read() # Ignore leading whitespace and comment lines. - deps = [ + constraints_deps = [ match.group(1) for match in re.finditer( r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE @@ -343,7 +373,7 @@ def prerelease(session): # We use --no-deps to ensure that pre-release versions aren't overwritten # by the version ranges in setup.py. - session.install(*deps) + session.install(*constraints_deps) session.install("--no-deps", "-e", ".[all]") # Print out prerelease package versions. @@ -368,6 +398,7 @@ def prerelease(session): @nox.session(python=DEFAULT_PYTHON_VERSION) +@_calculate_duration def cover(session): """Run the final coverage report. @@ -390,6 +421,7 @@ def cover(session): @nox.session(python="3.9") +@_calculate_duration def docs(session): """Build the docs for this library.""" @@ -425,6 +457,7 @@ def docs(session): @nox.session(python="3.10") +@_calculate_duration def docfx(session): """Build the docfx yaml files for this library.""" @@ -470,92 +503,6 @@ def docfx(session): ) -@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) -def prerelease_deps(session): - """Run all tests with prerelease versions of dependencies installed.""" - - # Install all dependencies - session.install("-e", ".[all, tests, tracing]") - unit_deps_all = UNIT_TEST_STANDARD_DEPENDENCIES + UNIT_TEST_EXTERNAL_DEPENDENCIES - session.install(*unit_deps_all) - system_deps_all = ( - SYSTEM_TEST_STANDARD_DEPENDENCIES + SYSTEM_TEST_EXTERNAL_DEPENDENCIES - ) - session.install(*system_deps_all) - - # Because we test minimum dependency versions on the minimum Python - # version, the first version we test with in the unit tests sessions has a - # constraints file containing all dependencies and extras. - with open( - CURRENT_DIRECTORY - / "testing" - / f"constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt", - encoding="utf-8", - ) as constraints_file: - constraints_text = constraints_file.read() - - # Ignore leading whitespace and comment lines. - constraints_deps = [ - match.group(1) - for match in re.finditer( - r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE - ) - ] - - session.install(*constraints_deps) - - prerel_deps = [ - # "protobuf", - # dependency of grpc - "six", - "googleapis-common-protos", - # Exclude version 1.52.0rc1 which has a known issue. See https://github.com/grpc/grpc/issues/32163 - "grpcio!=1.52.0rc1", - "grpcio-status", - "google-api-core", - "google-auth", - "proto-plus", - "google-cloud-testutils", - # dependencies of google-cloud-testutils" - "click", - ] - - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--upgrade", dep) - - # Remaining dependencies - other_deps = [ - "requests", - ] - session.install(*other_deps) - - # Print out package versions. - session.run("python", "-m", "pip", "freeze") - - session.run("py.test", "tests/unit") - - system_test_path = os.path.join("tests", "system.py") - system_test_folder_path = os.path.join("tests", "system") - - # Only run system tests if found. - if os.path.exists(system_test_path): - session.run( - "py.test", - "--verbose", - f"--junitxml=system_{session.python}_sponge_log.xml", - system_test_path, - *session.posargs, - ) - if os.path.exists(system_test_folder_path): - session.run( - "py.test", - "--verbose", - f"--junitxml=system_{session.python}_sponge_log.xml", - system_test_folder_path, - *session.posargs, - ) - - def install_conda_unittest_dependencies(session, standard_deps, conda_forge_packages): """Installs packages from conda forge, pypi, and locally.""" @@ -573,6 +520,7 @@ def install_conda_unittest_dependencies(session, standard_deps, conda_forge_pack @nox.session(python=CONDA_TEST_PYTHON_VERSIONS, venv_backend="mamba") +@_calculate_duration def conda_test(session): """Run test suite in a conda virtual environment. diff --git a/packages/pandas-gbq/owlbot.py b/packages/pandas-gbq/owlbot.py index af5348692a4a..3bb53fc2d16a 100644 --- a/packages/pandas-gbq/owlbot.py +++ b/packages/pandas-gbq/owlbot.py @@ -56,9 +56,13 @@ "docs/multiprocessing.rst", "noxfile.py", "README.rst", + # exclude .kokoro/build.sh which is customized due to support for conda ".kokoro/build.sh", - ], + + # exclude this file as we have an alternate prerelease.cfg + ".kokoro/presubmit/prerelease-deps.cfg", + ] ) # ---------------------------------------------------------------------------- From 661179f751dc8d5678b33e64d84162a8709d6f9c Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Wed, 12 Jun 2024 00:03:24 +0200 Subject: [PATCH 419/519] chore(deps): update all dependencies (#784) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update all dependencies * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot Co-authored-by: Lingqing Gan --- packages/pandas-gbq/samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 731d869db1c5..5e293661d385 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,6 +1,6 @@ google-cloud-bigquery-storage==2.25.0 google-cloud-bigquery==3.24.0 -pandas-gbq==0.23.0 +pandas-gbq==0.23.1 pandas===2.0.3; python_version == '3.8' pandas==2.2.2; python_version >= '3.9' pyarrow==16.1.0; python_version >= '3.8' From 9626fe98fa610b01f427f6d57801d1069a025e4b Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Sat, 22 Jun 2024 01:23:32 +0200 Subject: [PATCH 420/519] chore(deps): update all dependencies (#787) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update all dependencies * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot --- packages/pandas-gbq/samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 5e293661d385..cef89843b8c5 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,5 +1,5 @@ google-cloud-bigquery-storage==2.25.0 -google-cloud-bigquery==3.24.0 +google-cloud-bigquery==3.25.0 pandas-gbq==0.23.1 pandas===2.0.3; python_version == '3.8' pandas==2.2.2; python_version >= '3.9' From 26bb7f489cc30c8c9a583ec91795212487ba5e9c Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Wed, 3 Jul 2024 16:17:42 -0400 Subject: [PATCH 421/519] chore: update templated files (#789) Source-Link: https://github.com/googleapis/synthtool/commit/a37f74cd300d1f56d6f28c368d2931f72adee948 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:d3de8a02819f65001effcbd3ea76ce97e9bcff035c7a89457f40f892c87c5b32 Co-authored-by: Owl Bot --- packages/pandas-gbq/.coveragerc | 2 +- packages/pandas-gbq/.flake8 | 2 +- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 +- packages/pandas-gbq/.github/auto-label.yaml | 2 +- .../pandas-gbq/.kokoro/docker/docs/Dockerfile | 2 +- .../pandas-gbq/.kokoro/populate-secrets.sh | 2 +- packages/pandas-gbq/.kokoro/publish-docs.sh | 2 +- packages/pandas-gbq/.kokoro/release.sh | 2 +- packages/pandas-gbq/.kokoro/requirements.txt | 509 +++++++++--------- .../.kokoro/test-samples-against-head.sh | 2 +- .../pandas-gbq/.kokoro/test-samples-impl.sh | 2 +- packages/pandas-gbq/.kokoro/test-samples.sh | 2 +- packages/pandas-gbq/.kokoro/trampoline.sh | 2 +- packages/pandas-gbq/.kokoro/trampoline_v2.sh | 2 +- packages/pandas-gbq/.pre-commit-config.yaml | 2 +- packages/pandas-gbq/.trampolinerc | 2 +- packages/pandas-gbq/MANIFEST.in | 2 +- packages/pandas-gbq/docs/conf.py | 2 +- .../pandas-gbq/scripts/decrypt-secrets.sh | 2 +- .../scripts/readme-gen/readme_gen.py | 2 +- 20 files changed, 286 insertions(+), 263 deletions(-) diff --git a/packages/pandas-gbq/.coveragerc b/packages/pandas-gbq/.coveragerc index 6948e4d98345..b5577cddf390 100644 --- a/packages/pandas-gbq/.coveragerc +++ b/packages/pandas-gbq/.coveragerc @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/.flake8 b/packages/pandas-gbq/.flake8 index 87f6e408c47d..32986c79287a 100644 --- a/packages/pandas-gbq/.flake8 +++ b/packages/pandas-gbq/.flake8 @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 81f87c56917d..91d742b5b9fe 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:5a4c19d17e597b92d786e569be101e636c9c2817731f80a5adec56b2aa8fe070 -# created: 2024-04-12T11:35:58.922854369Z + digest: sha256:d3de8a02819f65001effcbd3ea76ce97e9bcff035c7a89457f40f892c87c5b32 +# created: 2024-07-03T17:43:00.77142528Z diff --git a/packages/pandas-gbq/.github/auto-label.yaml b/packages/pandas-gbq/.github/auto-label.yaml index 8b37ee89711f..21786a4eb085 100644 --- a/packages/pandas-gbq/.github/auto-label.yaml +++ b/packages/pandas-gbq/.github/auto-label.yaml @@ -1,4 +1,4 @@ -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile b/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile index bdaf39fe22d0..a26ce61930f5 100644 --- a/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile +++ b/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/.kokoro/populate-secrets.sh b/packages/pandas-gbq/.kokoro/populate-secrets.sh index 6f3972140e80..c435402f473e 100755 --- a/packages/pandas-gbq/.kokoro/populate-secrets.sh +++ b/packages/pandas-gbq/.kokoro/populate-secrets.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023 Google LLC. +# Copyright 2024 Google LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/.kokoro/publish-docs.sh b/packages/pandas-gbq/.kokoro/publish-docs.sh index 9eafe0be3bba..38f083f05aa0 100755 --- a/packages/pandas-gbq/.kokoro/publish-docs.sh +++ b/packages/pandas-gbq/.kokoro/publish-docs.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/.kokoro/release.sh b/packages/pandas-gbq/.kokoro/release.sh index 7502861622d5..1902895d1fc7 100755 --- a/packages/pandas-gbq/.kokoro/release.sh +++ b/packages/pandas-gbq/.kokoro/release.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/.kokoro/requirements.txt b/packages/pandas-gbq/.kokoro/requirements.txt index 51f92b8e12f1..35ece0e4d2e9 100644 --- a/packages/pandas-gbq/.kokoro/requirements.txt +++ b/packages/pandas-gbq/.kokoro/requirements.txt @@ -4,21 +4,25 @@ # # pip-compile --allow-unsafe --generate-hashes requirements.in # -argcomplete==3.1.4 \ - --hash=sha256:72558ba729e4c468572609817226fb0a6e7e9a0a7d477b882be168c0b4a62b94 \ - --hash=sha256:fbe56f8cda08aa9a04b307d8482ea703e96a6a801611acb4be9bf3942017989f +argcomplete==3.4.0 \ + --hash=sha256:69a79e083a716173e5532e0fa3bef45f793f4e61096cf52b5a42c0211c8b8aa5 \ + --hash=sha256:c2abcdfe1be8ace47ba777d4fce319eb13bf8ad9dace8d085dcad6eded88057f # via nox -attrs==23.1.0 \ - --hash=sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04 \ - --hash=sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015 +attrs==23.2.0 \ + --hash=sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30 \ + --hash=sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1 # via gcp-releasetool -cachetools==5.3.2 \ - --hash=sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2 \ - --hash=sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1 +backports-tarfile==1.2.0 \ + --hash=sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34 \ + --hash=sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991 + # via jaraco-context +cachetools==5.3.3 \ + --hash=sha256:0abad1021d3f8325b2fc1d2e9c8b9c9d57b04c3932657a72465447332c24d945 \ + --hash=sha256:ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105 # via google-auth -certifi==2023.7.22 \ - --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 \ - --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9 +certifi==2024.6.2 \ + --hash=sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516 \ + --hash=sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56 # via requests cffi==1.16.0 \ --hash=sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc \ @@ -87,90 +91,90 @@ click==8.0.4 \ # -r requirements.in # gcp-docuploader # gcp-releasetool -colorlog==6.7.0 \ - --hash=sha256:0d33ca236784a1ba3ff9c532d4964126d8a2c44f1f0cb1d2b0728196f512f662 \ - --hash=sha256:bd94bd21c1e13fac7bd3153f4bc3a7dc0eb0974b8bc2fdf1a989e474f6e582e5 +colorlog==6.8.2 \ + --hash=sha256:3e3e079a41feb5a1b64f978b5ea4f46040a94f11f0e8bbb8261e3dbbeca64d44 \ + --hash=sha256:4dcbb62368e2800cb3c5abd348da7e53f6c362dda502ec27c560b2e58a66bd33 # via # gcp-docuploader # nox -cryptography==42.0.5 \ - --hash=sha256:0270572b8bd2c833c3981724b8ee9747b3ec96f699a9665470018594301439ee \ - --hash=sha256:111a0d8553afcf8eb02a4fea6ca4f59d48ddb34497aa8706a6cf536f1a5ec576 \ - --hash=sha256:16a48c23a62a2f4a285699dba2e4ff2d1cff3115b9df052cdd976a18856d8e3d \ - --hash=sha256:1b95b98b0d2af784078fa69f637135e3c317091b615cd0905f8b8a087e86fa30 \ - --hash=sha256:1f71c10d1e88467126f0efd484bd44bca5e14c664ec2ede64c32f20875c0d413 \ - --hash=sha256:2424ff4c4ac7f6b8177b53c17ed5d8fa74ae5955656867f5a8affaca36a27abb \ - --hash=sha256:2bce03af1ce5a5567ab89bd90d11e7bbdff56b8af3acbbec1faded8f44cb06da \ - --hash=sha256:329906dcc7b20ff3cad13c069a78124ed8247adcac44b10bea1130e36caae0b4 \ - --hash=sha256:37dd623507659e08be98eec89323469e8c7b4c1407c85112634ae3dbdb926fdd \ - --hash=sha256:3eaafe47ec0d0ffcc9349e1708be2aaea4c6dd4978d76bf6eb0cb2c13636c6fc \ - --hash=sha256:5e6275c09d2badf57aea3afa80d975444f4be8d3bc58f7f80d2a484c6f9485c8 \ - --hash=sha256:6fe07eec95dfd477eb9530aef5bead34fec819b3aaf6c5bd6d20565da607bfe1 \ - --hash=sha256:7367d7b2eca6513681127ebad53b2582911d1736dc2ffc19f2c3ae49997496bc \ - --hash=sha256:7cde5f38e614f55e28d831754e8a3bacf9ace5d1566235e39d91b35502d6936e \ - --hash=sha256:9481ffe3cf013b71b2428b905c4f7a9a4f76ec03065b05ff499bb5682a8d9ad8 \ - --hash=sha256:98d8dc6d012b82287f2c3d26ce1d2dd130ec200c8679b6213b3c73c08b2b7940 \ - --hash=sha256:a011a644f6d7d03736214d38832e030d8268bcff4a41f728e6030325fea3e400 \ - --hash=sha256:a2913c5375154b6ef2e91c10b5720ea6e21007412f6437504ffea2109b5a33d7 \ - --hash=sha256:a30596bae9403a342c978fb47d9b0ee277699fa53bbafad14706af51fe543d16 \ - --hash=sha256:b03c2ae5d2f0fc05f9a2c0c997e1bc18c8229f392234e8a0194f202169ccd278 \ - --hash=sha256:b6cd2203306b63e41acdf39aa93b86fb566049aeb6dc489b70e34bcd07adca74 \ - --hash=sha256:b7ffe927ee6531c78f81aa17e684e2ff617daeba7f189f911065b2ea2d526dec \ - --hash=sha256:b8cac287fafc4ad485b8a9b67d0ee80c66bf3574f655d3b97ef2e1082360faf1 \ - --hash=sha256:ba334e6e4b1d92442b75ddacc615c5476d4ad55cc29b15d590cc6b86efa487e2 \ - --hash=sha256:ba3e4a42397c25b7ff88cdec6e2a16c2be18720f317506ee25210f6d31925f9c \ - --hash=sha256:c41fb5e6a5fe9ebcd58ca3abfeb51dffb5d83d6775405305bfa8715b76521922 \ - --hash=sha256:cd2030f6650c089aeb304cf093f3244d34745ce0cfcc39f20c6fbfe030102e2a \ - --hash=sha256:cd65d75953847815962c84a4654a84850b2bb4aed3f26fadcc1c13892e1e29f6 \ - --hash=sha256:e4985a790f921508f36f81831817cbc03b102d643b5fcb81cd33df3fa291a1a1 \ - --hash=sha256:e807b3188f9eb0eaa7bbb579b462c5ace579f1cedb28107ce8b48a9f7ad3679e \ - --hash=sha256:f12764b8fffc7a123f641d7d049d382b73f96a34117e0b637b80643169cec8ac \ - --hash=sha256:f8837fe1d6ac4a8052a9a8ddab256bc006242696f03368a4009be7ee3075cdb7 +cryptography==42.0.8 \ + --hash=sha256:013629ae70b40af70c9a7a5db40abe5d9054e6f4380e50ce769947b73bf3caad \ + --hash=sha256:2346b911eb349ab547076f47f2e035fc8ff2c02380a7cbbf8d87114fa0f1c583 \ + --hash=sha256:2f66d9cd9147ee495a8374a45ca445819f8929a3efcd2e3df6428e46c3cbb10b \ + --hash=sha256:2f88d197e66c65be5e42cd72e5c18afbfae3f741742070e3019ac8f4ac57262c \ + --hash=sha256:31f721658a29331f895a5a54e7e82075554ccfb8b163a18719d342f5ffe5ecb1 \ + --hash=sha256:343728aac38decfdeecf55ecab3264b015be68fc2816ca800db649607aeee648 \ + --hash=sha256:5226d5d21ab681f432a9c1cf8b658c0cb02533eece706b155e5fbd8a0cdd3949 \ + --hash=sha256:57080dee41209e556a9a4ce60d229244f7a66ef52750f813bfbe18959770cfba \ + --hash=sha256:5a94eccb2a81a309806027e1670a358b99b8fe8bfe9f8d329f27d72c094dde8c \ + --hash=sha256:6b7c4f03ce01afd3b76cf69a5455caa9cfa3de8c8f493e0d3ab7d20611c8dae9 \ + --hash=sha256:7016f837e15b0a1c119d27ecd89b3515f01f90a8615ed5e9427e30d9cdbfed3d \ + --hash=sha256:81884c4d096c272f00aeb1f11cf62ccd39763581645b0812e99a91505fa48e0c \ + --hash=sha256:81d8a521705787afe7a18d5bfb47ea9d9cc068206270aad0b96a725022e18d2e \ + --hash=sha256:8d09d05439ce7baa8e9e95b07ec5b6c886f548deb7e0f69ef25f64b3bce842f2 \ + --hash=sha256:961e61cefdcb06e0c6d7e3a1b22ebe8b996eb2bf50614e89384be54c48c6b63d \ + --hash=sha256:9c0c1716c8447ee7dbf08d6db2e5c41c688544c61074b54fc4564196f55c25a7 \ + --hash=sha256:a0608251135d0e03111152e41f0cc2392d1e74e35703960d4190b2e0f4ca9c70 \ + --hash=sha256:a0c5b2b0585b6af82d7e385f55a8bc568abff8923af147ee3c07bd8b42cda8b2 \ + --hash=sha256:ad803773e9df0b92e0a817d22fd8a3675493f690b96130a5e24f1b8fabbea9c7 \ + --hash=sha256:b297f90c5723d04bcc8265fc2a0f86d4ea2e0f7ab4b6994459548d3a6b992a14 \ + --hash=sha256:ba4f0a211697362e89ad822e667d8d340b4d8d55fae72cdd619389fb5912eefe \ + --hash=sha256:c4783183f7cb757b73b2ae9aed6599b96338eb957233c58ca8f49a49cc32fd5e \ + --hash=sha256:c9bb2ae11bfbab395bdd072985abde58ea9860ed84e59dbc0463a5d0159f5b71 \ + --hash=sha256:cafb92b2bc622cd1aa6a1dce4b93307792633f4c5fe1f46c6b97cf67073ec961 \ + --hash=sha256:d45b940883a03e19e944456a558b67a41160e367a719833c53de6911cabba2b7 \ + --hash=sha256:dc0fdf6787f37b1c6b08e6dfc892d9d068b5bdb671198c72072828b80bd5fe4c \ + --hash=sha256:dea567d1b0e8bc5764b9443858b673b734100c2871dc93163f58c46a97a83d28 \ + --hash=sha256:dec9b018df185f08483f294cae6ccac29e7a6e0678996587363dc352dc65c842 \ + --hash=sha256:e3ec3672626e1b9e55afd0df6d774ff0e953452886e06e0f1eb7eb0c832e8902 \ + --hash=sha256:e599b53fd95357d92304510fb7bda8523ed1f79ca98dce2f43c115950aa78801 \ + --hash=sha256:fa76fbb7596cc5839320000cdd5d0955313696d9511debab7ee7278fc8b5c84a \ + --hash=sha256:fff12c88a672ab9c9c1cf7b0c80e3ad9e2ebd9d828d955c126be4fd3e5578c9e # via # -r requirements.in # gcp-releasetool # secretstorage -distlib==0.3.7 \ - --hash=sha256:2e24928bc811348f0feb63014e97aaae3037f2cf48712d51ae61df7fd6075057 \ - --hash=sha256:9dafe54b34a028eafd95039d5e5d4851a13734540f1331060d31c9916e7147a8 +distlib==0.3.8 \ + --hash=sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784 \ + --hash=sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64 # via virtualenv -docutils==0.20.1 \ - --hash=sha256:96f387a2c5562db4476f09f13bbab2192e764cac08ebbf3a34a95d9b1e4a59d6 \ - --hash=sha256:f08a4e276c3a1583a86dce3e34aba3fe04d02bba2dd51ed16106244e8a923e3b +docutils==0.21.2 \ + --hash=sha256:3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f \ + --hash=sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2 # via readme-renderer -filelock==3.13.1 \ - --hash=sha256:521f5f56c50f8426f5e03ad3b281b490a87ef15bc6c526f168290f0c7148d44e \ - --hash=sha256:57dbda9b35157b05fb3e58ee91448612eb674172fab98ee235ccb0b5bee19a1c +filelock==3.15.4 \ + --hash=sha256:2207938cbc1844345cb01a5a95524dae30f0ce089eba5b00378295a17e3e90cb \ + --hash=sha256:6ca1fffae96225dab4c6eaf1c4f4f28cd2568d3ec2a44e15a08520504de468e7 # via virtualenv gcp-docuploader==0.6.5 \ --hash=sha256:30221d4ac3e5a2b9c69aa52fdbef68cc3f27d0e6d0d90e220fc024584b8d2318 \ --hash=sha256:b7458ef93f605b9d46a4bf3a8dc1755dad1f31d030c8679edf304e343b347eea # via -r requirements.in -gcp-releasetool==2.0.0 \ - --hash=sha256:3d73480b50ba243f22d7c7ec08b115a30e1c7817c4899781840c26f9c55b8277 \ - --hash=sha256:7aa9fd935ec61e581eb8458ad00823786d91756c25e492f372b2b30962f3c28f +gcp-releasetool==2.0.1 \ + --hash=sha256:34314a910c08e8911d9c965bd44f8f2185c4f556e737d719c33a41f6a610de96 \ + --hash=sha256:b0d5863c6a070702b10883d37c4bdfd74bf930fe417f36c0c965d3b7c779ae62 # via -r requirements.in -google-api-core==2.12.0 \ - --hash=sha256:c22e01b1e3c4dcd90998494879612c38d0a3411d1f7b679eb89e2abe3ce1f553 \ - --hash=sha256:ec6054f7d64ad13b41e43d96f735acbd763b0f3b695dabaa2d579673f6a6e160 +google-api-core==2.19.1 \ + --hash=sha256:f12a9b8309b5e21d92483bbd47ce2c445861ec7d269ef6784ecc0ea8c1fa6125 \ + --hash=sha256:f4695f1e3650b316a795108a76a1c416e6afb036199d1c1f1f110916df479ffd # via # google-cloud-core # google-cloud-storage -google-auth==2.23.4 \ - --hash=sha256:79905d6b1652187def79d491d6e23d0cbb3a21d3c7ba0dbaa9c8a01906b13ff3 \ - --hash=sha256:d4bbc92fe4b8bfd2f3e8d88e5ba7085935da208ee38a134fc280e7ce682a05f2 +google-auth==2.31.0 \ + --hash=sha256:042c4702efa9f7d3c48d3a69341c209381b125faa6dbf3ebe56bc7e40ae05c23 \ + --hash=sha256:87805c36970047247c8afe614d4e3af8eceafc1ebba0c679fe75ddd1d575e871 # via # gcp-releasetool # google-api-core # google-cloud-core # google-cloud-storage -google-cloud-core==2.3.3 \ - --hash=sha256:37b80273c8d7eee1ae816b3a20ae43585ea50506cb0e60f3cf5be5f87f1373cb \ - --hash=sha256:fbd11cad3e98a7e5b0343dc07cb1039a5ffd7a5bb96e1f1e27cee4bda4a90863 +google-cloud-core==2.4.1 \ + --hash=sha256:9b7749272a812bde58fff28868d0c5e2f585b82f37e09a1f6ed2d4d10f134073 \ + --hash=sha256:a9e6a4422b9ac5c29f79a0ede9485473338e2ce78d91f2370c01e730eab22e61 # via google-cloud-storage -google-cloud-storage==2.13.0 \ - --hash=sha256:ab0bf2e1780a1b74cf17fccb13788070b729f50c252f0c94ada2aae0ca95437d \ - --hash=sha256:f62dc4c7b6cd4360d072e3deb28035fbdad491ac3d9b0b1815a12daea10f37c7 +google-cloud-storage==2.17.0 \ + --hash=sha256:49378abff54ef656b52dca5ef0f2eba9aa83dc2b2c72c78714b03a1a95fe9388 \ + --hash=sha256:5b393bc766b7a3bc6f5407b9e665b2450d36282614b7945e570b3480a456d1e1 # via gcp-docuploader google-crc32c==1.5.0 \ --hash=sha256:024894d9d3cfbc5943f8f230e23950cd4906b2fe004c72e29b209420a1e6b05a \ @@ -244,28 +248,36 @@ google-crc32c==1.5.0 \ # via # google-cloud-storage # google-resumable-media -google-resumable-media==2.6.0 \ - --hash=sha256:972852f6c65f933e15a4a210c2b96930763b47197cdf4aa5f5bea435efb626e7 \ - --hash=sha256:fc03d344381970f79eebb632a3c18bb1828593a2dc5572b5f90115ef7d11e81b +google-resumable-media==2.7.1 \ + --hash=sha256:103ebc4ba331ab1bfdac0250f8033627a2cd7cde09e7ccff9181e31ba4315b2c \ + --hash=sha256:eae451a7b2e2cdbaaa0fd2eb00cc8a1ee5e95e16b55597359cbc3d27d7d90e33 # via google-cloud-storage -googleapis-common-protos==1.61.0 \ - --hash=sha256:22f1915393bb3245343f6efe87f6fe868532efc12aa26b391b15132e1279f1c0 \ - --hash=sha256:8a64866a97f6304a7179873a465d6eee97b7a24ec6cfd78e0f575e96b821240b +googleapis-common-protos==1.63.2 \ + --hash=sha256:27a2499c7e8aff199665b22741997e485eccc8645aa9176c7c988e6fae507945 \ + --hash=sha256:27c5abdffc4911f28101e635de1533fb4cfd2c37fbaa9174587c799fac90aa87 # via google-api-core idna==3.7 \ --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \ --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0 # via requests -importlib-metadata==6.8.0 \ - --hash=sha256:3ebb78df84a805d7698245025b975d9d67053cd94c79245ba4b3eb694abe68bb \ - --hash=sha256:dbace7892d8c0c4ac1ad096662232f831d4e64f4c4545bd53016a3e9d4654743 +importlib-metadata==8.0.0 \ + --hash=sha256:15584cf2b1bf449d98ff8a6ff1abef57bf20f3ac6454f431736cd3e660921b2f \ + --hash=sha256:188bd24e4c346d3f0a933f275c2fec67050326a856b9a359881d7c2a697e8812 # via # -r requirements.in # keyring # twine -jaraco-classes==3.3.0 \ - --hash=sha256:10afa92b6743f25c0cf5f37c6bb6e18e2c5bb84a16527ccfc0040ea377e7aaeb \ - --hash=sha256:c063dd08e89217cee02c8d5e5ec560f2c8ce6cdc2fcdc2e68f7b2e5547ed3621 +jaraco-classes==3.4.0 \ + --hash=sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd \ + --hash=sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790 + # via keyring +jaraco-context==5.3.0 \ + --hash=sha256:3e16388f7da43d384a1a7cd3452e72e14732ac9fe459678773a3608a812bf266 \ + --hash=sha256:c2f67165ce1f9be20f32f650f25d8edfc1646a8aeee48ae06fb35f90763576d2 + # via keyring +jaraco-functools==4.0.1 \ + --hash=sha256:3b24ccb921d6b593bdceb56ce14799204f473976e2a9d4b15b04d0f2c2326664 \ + --hash=sha256:d33fa765374c0611b52f8b3a795f8900869aa88c84769d4d1746cd68fb28c3e8 # via keyring jeepney==0.8.0 \ --hash=sha256:5efe48d255973902f6badc3ce55e2aa6c5c3b3bc642059ef3a91247bcfcc5806 \ @@ -273,13 +285,13 @@ jeepney==0.8.0 \ # via # keyring # secretstorage -jinja2==3.1.3 \ - --hash=sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa \ - --hash=sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90 +jinja2==3.1.4 \ + --hash=sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369 \ + --hash=sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d # via gcp-releasetool -keyring==24.2.0 \ - --hash=sha256:4901caaf597bfd3bbd78c9a0c7c4c29fcd8310dab2cffefe749e916b6527acd6 \ - --hash=sha256:ca0746a19ec421219f4d713f848fa297a661a8a8c1504867e55bfb5e09091509 +keyring==25.2.1 \ + --hash=sha256:2458681cdefc0dbc0b7eb6cf75d0b98e59f9ad9b2d4edd319d18f68bdca95e50 \ + --hash=sha256:daaffd42dbda25ddafb1ad5fec4024e5bbcfe424597ca1ca452b299861e49f1b # via # gcp-releasetool # twine @@ -287,146 +299,153 @@ markdown-it-py==3.0.0 \ --hash=sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1 \ --hash=sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb # via rich -markupsafe==2.1.3 \ - --hash=sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e \ - --hash=sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e \ - --hash=sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431 \ - --hash=sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686 \ - --hash=sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c \ - --hash=sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559 \ - --hash=sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc \ - --hash=sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb \ - --hash=sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939 \ - --hash=sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c \ - --hash=sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0 \ - --hash=sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4 \ - --hash=sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9 \ - --hash=sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575 \ - --hash=sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba \ - --hash=sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d \ - --hash=sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd \ - --hash=sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3 \ - --hash=sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00 \ - --hash=sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155 \ - --hash=sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac \ - --hash=sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52 \ - --hash=sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f \ - --hash=sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8 \ - --hash=sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b \ - --hash=sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007 \ - --hash=sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24 \ - --hash=sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea \ - --hash=sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198 \ - --hash=sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0 \ - --hash=sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee \ - --hash=sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be \ - --hash=sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2 \ - --hash=sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1 \ - --hash=sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707 \ - --hash=sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6 \ - --hash=sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c \ - --hash=sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58 \ - --hash=sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823 \ - --hash=sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779 \ - --hash=sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636 \ - --hash=sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c \ - --hash=sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad \ - --hash=sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee \ - --hash=sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc \ - --hash=sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2 \ - --hash=sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48 \ - --hash=sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7 \ - --hash=sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e \ - --hash=sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b \ - --hash=sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa \ - --hash=sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5 \ - --hash=sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e \ - --hash=sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb \ - --hash=sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9 \ - --hash=sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57 \ - --hash=sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc \ - --hash=sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc \ - --hash=sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2 \ - --hash=sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11 +markupsafe==2.1.5 \ + --hash=sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf \ + --hash=sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff \ + --hash=sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f \ + --hash=sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3 \ + --hash=sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532 \ + --hash=sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f \ + --hash=sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617 \ + --hash=sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df \ + --hash=sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4 \ + --hash=sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906 \ + --hash=sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f \ + --hash=sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4 \ + --hash=sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8 \ + --hash=sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371 \ + --hash=sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2 \ + --hash=sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465 \ + --hash=sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52 \ + --hash=sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6 \ + --hash=sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169 \ + --hash=sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad \ + --hash=sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2 \ + --hash=sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0 \ + --hash=sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029 \ + --hash=sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f \ + --hash=sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a \ + --hash=sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced \ + --hash=sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5 \ + --hash=sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c \ + --hash=sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf \ + --hash=sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9 \ + --hash=sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb \ + --hash=sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad \ + --hash=sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3 \ + --hash=sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1 \ + --hash=sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46 \ + --hash=sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc \ + --hash=sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a \ + --hash=sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee \ + --hash=sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900 \ + --hash=sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5 \ + --hash=sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea \ + --hash=sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f \ + --hash=sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5 \ + --hash=sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e \ + --hash=sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a \ + --hash=sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f \ + --hash=sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50 \ + --hash=sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a \ + --hash=sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b \ + --hash=sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4 \ + --hash=sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff \ + --hash=sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2 \ + --hash=sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46 \ + --hash=sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b \ + --hash=sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf \ + --hash=sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5 \ + --hash=sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5 \ + --hash=sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab \ + --hash=sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd \ + --hash=sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68 # via jinja2 mdurl==0.1.2 \ --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba # via markdown-it-py -more-itertools==10.1.0 \ - --hash=sha256:626c369fa0eb37bac0291bce8259b332fd59ac792fa5497b59837309cd5b114a \ - --hash=sha256:64e0735fcfdc6f3464ea133afe8ea4483b1c5fe3a3d69852e6503b43a0b222e6 - # via jaraco-classes -nh3==0.2.14 \ - --hash=sha256:116c9515937f94f0057ef50ebcbcc10600860065953ba56f14473ff706371873 \ - --hash=sha256:18415df36db9b001f71a42a3a5395db79cf23d556996090d293764436e98e8ad \ - --hash=sha256:203cac86e313cf6486704d0ec620a992c8bc164c86d3a4fd3d761dd552d839b5 \ - --hash=sha256:2b0be5c792bd43d0abef8ca39dd8acb3c0611052ce466d0401d51ea0d9aa7525 \ - --hash=sha256:377aaf6a9e7c63962f367158d808c6a1344e2b4f83d071c43fbd631b75c4f0b2 \ - --hash=sha256:525846c56c2bcd376f5eaee76063ebf33cf1e620c1498b2a40107f60cfc6054e \ - --hash=sha256:5529a3bf99402c34056576d80ae5547123f1078da76aa99e8ed79e44fa67282d \ - --hash=sha256:7771d43222b639a4cd9e341f870cee336b9d886de1ad9bec8dddab22fe1de450 \ - --hash=sha256:88c753efbcdfc2644a5012938c6b9753f1c64a5723a67f0301ca43e7b85dcf0e \ - --hash=sha256:93a943cfd3e33bd03f77b97baa11990148687877b74193bf777956b67054dcc6 \ - --hash=sha256:9be2f68fb9a40d8440cbf34cbf40758aa7f6093160bfc7fb018cce8e424f0c3a \ - --hash=sha256:a0c509894fd4dccdff557068e5074999ae3b75f4c5a2d6fb5415e782e25679c4 \ - --hash=sha256:ac8056e937f264995a82bf0053ca898a1cb1c9efc7cd68fa07fe0060734df7e4 \ - --hash=sha256:aed56a86daa43966dd790ba86d4b810b219f75b4bb737461b6886ce2bde38fd6 \ - --hash=sha256:e8986f1dd3221d1e741fda0a12eaa4a273f1d80a35e31a1ffe579e7c621d069e \ - --hash=sha256:f99212a81c62b5f22f9e7c3e347aa00491114a5647e1f13bbebd79c3e5f08d75 +more-itertools==10.3.0 \ + --hash=sha256:e5d93ef411224fbcef366a6e8ddc4c5781bc6359d43412a65dd5964e46111463 \ + --hash=sha256:ea6a02e24a9161e51faad17a8782b92a0df82c12c1c8886fec7f0c3fa1a1b320 + # via + # jaraco-classes + # jaraco-functools +nh3==0.2.17 \ + --hash=sha256:0316c25b76289cf23be6b66c77d3608a4fdf537b35426280032f432f14291b9a \ + --hash=sha256:1a814dd7bba1cb0aba5bcb9bebcc88fd801b63e21e2450ae6c52d3b3336bc911 \ + --hash=sha256:1aa52a7def528297f256de0844e8dd680ee279e79583c76d6fa73a978186ddfb \ + --hash=sha256:22c26e20acbb253a5bdd33d432a326d18508a910e4dcf9a3316179860d53345a \ + --hash=sha256:40015514022af31975c0b3bca4014634fa13cb5dc4dbcbc00570acc781316dcc \ + --hash=sha256:40d0741a19c3d645e54efba71cb0d8c475b59135c1e3c580f879ad5514cbf028 \ + --hash=sha256:551672fd71d06cd828e282abdb810d1be24e1abb7ae2543a8fa36a71c1006fe9 \ + --hash=sha256:66f17d78826096291bd264f260213d2b3905e3c7fae6dfc5337d49429f1dc9f3 \ + --hash=sha256:85cdbcca8ef10733bd31f931956f7fbb85145a4d11ab9e6742bbf44d88b7e351 \ + --hash=sha256:a3f55fabe29164ba6026b5ad5c3151c314d136fd67415a17660b4aaddacf1b10 \ + --hash=sha256:b4427ef0d2dfdec10b641ed0bdaf17957eb625b2ec0ea9329b3d28806c153d71 \ + --hash=sha256:ba73a2f8d3a1b966e9cdba7b211779ad8a2561d2dba9674b8a19ed817923f65f \ + --hash=sha256:c21bac1a7245cbd88c0b0e4a420221b7bfa838a2814ee5bb924e9c2f10a1120b \ + --hash=sha256:c551eb2a3876e8ff2ac63dff1585236ed5dfec5ffd82216a7a174f7c5082a78a \ + --hash=sha256:c790769152308421283679a142dbdb3d1c46c79c823008ecea8e8141db1a2062 \ + --hash=sha256:d7a25fd8c86657f5d9d576268e3b3767c5cd4f42867c9383618be8517f0f022a # via readme-renderer -nox==2023.4.22 \ - --hash=sha256:0b1adc619c58ab4fa57d6ab2e7823fe47a32e70202f287d78474adcc7bda1891 \ - --hash=sha256:46c0560b0dc609d7d967dc99e22cb463d3c4caf54a5fda735d6c11b5177e3a9f +nox==2024.4.15 \ + --hash=sha256:6492236efa15a460ecb98e7b67562a28b70da006ab0be164e8821177577c0565 \ + --hash=sha256:ecf6700199cdfa9e5ea0a41ff5e6ef4641d09508eda6edb89d9987864115817f # via -r requirements.in -packaging==23.2 \ - --hash=sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5 \ - --hash=sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7 +packaging==24.1 \ + --hash=sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002 \ + --hash=sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124 # via # gcp-releasetool # nox -pkginfo==1.9.6 \ - --hash=sha256:4b7a555a6d5a22169fcc9cf7bfd78d296b0361adad412a346c1226849af5e546 \ - --hash=sha256:8fd5896e8718a4372f0ea9cc9d96f6417c9b986e23a4d116dda26b62cc29d046 +pkginfo==1.10.0 \ + --hash=sha256:5df73835398d10db79f8eecd5cd86b1f6d29317589ea70796994d49399af6297 \ + --hash=sha256:889a6da2ed7ffc58ab5b900d888ddce90bce912f2d2de1dc1c26f4cb9fe65097 # via twine -platformdirs==3.11.0 \ - --hash=sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3 \ - --hash=sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e +platformdirs==4.2.2 \ + --hash=sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee \ + --hash=sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3 # via virtualenv -protobuf==4.25.3 \ - --hash=sha256:19b270aeaa0099f16d3ca02628546b8baefe2955bbe23224aaf856134eccf1e4 \ - --hash=sha256:209ba4cc916bab46f64e56b85b090607a676f66b473e6b762e6f1d9d591eb2e8 \ - --hash=sha256:25b5d0b42fd000320bd7830b349e3b696435f3b329810427a6bcce6a5492cc5c \ - --hash=sha256:7c8daa26095f82482307bc717364e7c13f4f1c99659be82890dcfc215194554d \ - --hash=sha256:c053062984e61144385022e53678fbded7aea14ebb3e0305ae3592fb219ccfa4 \ - --hash=sha256:d4198877797a83cbfe9bffa3803602bbe1625dc30d8a097365dbc762e5790faa \ - --hash=sha256:e3c97a1555fd6388f857770ff8b9703083de6bf1f9274a002a332d65fbb56c8c \ - --hash=sha256:e7cb0ae90dd83727f0c0718634ed56837bfeeee29a5f82a7514c03ee1364c019 \ - --hash=sha256:f0700d54bcf45424477e46a9f0944155b46fb0639d69728739c0e47bab83f2b9 \ - --hash=sha256:f1279ab38ecbfae7e456a108c5c0681e4956d5b1090027c1de0f934dfdb4b35c \ - --hash=sha256:f4f118245c4a087776e0a8408be33cf09f6c547442c00395fbfb116fac2f8ac2 +proto-plus==1.24.0 \ + --hash=sha256:30b72a5ecafe4406b0d339db35b56c4059064e69227b8c3bda7462397f966445 \ + --hash=sha256:402576830425e5f6ce4c2a6702400ac79897dab0b4343821aa5188b0fab81a12 + # via google-api-core +protobuf==5.27.2 \ + --hash=sha256:0e341109c609749d501986b835f667c6e1e24531096cff9d34ae411595e26505 \ + --hash=sha256:176c12b1f1c880bf7a76d9f7c75822b6a2bc3db2d28baa4d300e8ce4cde7409b \ + --hash=sha256:354d84fac2b0d76062e9b3221f4abbbacdfd2a4d8af36bab0474f3a0bb30ab38 \ + --hash=sha256:4fadd8d83e1992eed0248bc50a4a6361dc31bcccc84388c54c86e530b7f58863 \ + --hash=sha256:54330f07e4949d09614707c48b06d1a22f8ffb5763c159efd5c0928326a91470 \ + --hash=sha256:610e700f02469c4a997e58e328cac6f305f649826853813177e6290416e846c6 \ + --hash=sha256:7fc3add9e6003e026da5fc9e59b131b8f22b428b991ccd53e2af8071687b4fce \ + --hash=sha256:9e8f199bf7f97bd7ecebffcae45ebf9527603549b2b562df0fbc6d4d688f14ca \ + --hash=sha256:a109916aaac42bff84702fb5187f3edadbc7c97fc2c99c5ff81dd15dcce0d1e5 \ + --hash=sha256:b848dbe1d57ed7c191dfc4ea64b8b004a3f9ece4bf4d0d80a367b76df20bf36e \ + --hash=sha256:f3ecdef226b9af856075f28227ff2c90ce3a594d092c39bee5513573f25e2714 # via # gcp-docuploader # gcp-releasetool # google-api-core # googleapis-common-protos -pyasn1==0.5.0 \ - --hash=sha256:87a2121042a1ac9358cabcaf1d07680ff97ee6404333bacca15f76aa8ad01a57 \ - --hash=sha256:97b7290ca68e62a832558ec3976f15cbf911bf5d7c7039d8b861c2a0ece69fde + # proto-plus +pyasn1==0.6.0 \ + --hash=sha256:3a35ab2c4b5ef98e17dfdec8ab074046fbda76e281c5a706ccd82328cfc8f64c \ + --hash=sha256:cca4bb0f2df5504f02f6f8a775b6e416ff9b0b3b16f7ee80b5a3153d9b804473 # via # pyasn1-modules # rsa -pyasn1-modules==0.3.0 \ - --hash=sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c \ - --hash=sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d +pyasn1-modules==0.4.0 \ + --hash=sha256:831dbcea1b177b28c9baddf4c6d1013c24c3accd14a1873fffaa6a2e905f17b6 \ + --hash=sha256:be04f15b66c206eed667e0bb5ab27e2b1855ea54a842e5037738099e8ca4ae0b # via google-auth -pycparser==2.21 \ - --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ - --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206 +pycparser==2.22 \ + --hash=sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6 \ + --hash=sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc # via cffi -pygments==2.16.1 \ - --hash=sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692 \ - --hash=sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29 +pygments==2.18.0 \ + --hash=sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199 \ + --hash=sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a # via # readme-renderer # rich @@ -434,20 +453,20 @@ pyjwt==2.8.0 \ --hash=sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de \ --hash=sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320 # via gcp-releasetool -pyperclip==1.8.2 \ - --hash=sha256:105254a8b04934f0bc84e9c24eb360a591aaf6535c9def5f29d92af107a9bf57 +pyperclip==1.9.0 \ + --hash=sha256:b7de0142ddc81bfc5c7507eea19da920b92252b548b96186caf94a5e2527d310 # via gcp-releasetool -python-dateutil==2.8.2 \ - --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ - --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 +python-dateutil==2.9.0.post0 \ + --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ + --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 # via gcp-releasetool -readme-renderer==42.0 \ - --hash=sha256:13d039515c1f24de668e2c93f2e877b9dbe6c6c32328b90a40a49d8b2b85f36d \ - --hash=sha256:2d55489f83be4992fe4454939d1a051c33edbab778e82761d060c9fc6b308cd1 +readme-renderer==43.0 \ + --hash=sha256:1818dd28140813509eeed8d62687f7cd4f7bad90d4db586001c5dc09d4fde311 \ + --hash=sha256:19db308d86ecd60e5affa3b2a98f017af384678c63c88e5d4556a380e674f3f9 # via twine -requests==2.31.0 \ - --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \ - --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1 +requests==2.32.3 \ + --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \ + --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 # via # gcp-releasetool # google-api-core @@ -462,9 +481,9 @@ rfc3986==2.0.0 \ --hash=sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd \ --hash=sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c # via twine -rich==13.6.0 \ - --hash=sha256:2b38e2fe9ca72c9a00170a1a2d20c63c790d0e10ef1fe35eba76e1e7b1d7d245 \ - --hash=sha256:5c14d22737e6d5084ef4771b62d5d4363165b403455a30a1c8ca39dc7b644bef +rich==13.7.1 \ + --hash=sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222 \ + --hash=sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432 # via twine rsa==4.9 \ --hash=sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7 \ @@ -480,35 +499,39 @@ six==1.16.0 \ # via # gcp-docuploader # python-dateutil -twine==4.0.2 \ - --hash=sha256:929bc3c280033347a00f847236564d1c52a3e61b1ac2516c97c48f3ceab756d8 \ - --hash=sha256:9e102ef5fdd5a20661eb88fad46338806c3bd32cf1db729603fe3697b1bc83c8 +tomli==2.0.1 \ + --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \ + --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f + # via nox +twine==5.1.1 \ + --hash=sha256:215dbe7b4b94c2c50a7315c0275d2258399280fbb7d04182c7e55e24b5f93997 \ + --hash=sha256:9aa0825139c02b3434d913545c7b847a21c835e11597f5255842d457da2322db # via -r requirements.in -typing-extensions==4.8.0 \ - --hash=sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0 \ - --hash=sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef +typing-extensions==4.12.2 \ + --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \ + --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8 # via -r requirements.in -urllib3==2.0.7 \ - --hash=sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84 \ - --hash=sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e +urllib3==2.2.2 \ + --hash=sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472 \ + --hash=sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168 # via # requests # twine -virtualenv==20.24.6 \ - --hash=sha256:02ece4f56fbf939dbbc33c0715159951d6bf14aaf5457b092e4548e1382455af \ - --hash=sha256:520d056652454c5098a00c0f073611ccbea4c79089331f60bf9d7ba247bb7381 +virtualenv==20.26.3 \ + --hash=sha256:4c43a2a236279d9ea36a0d76f98d84bd6ca94ac4e0f4a3b9d46d05e10fea542a \ + --hash=sha256:8cc4a31139e796e9a7de2cd5cf2489de1217193116a8fd42328f1bd65f434589 # via nox -wheel==0.41.3 \ - --hash=sha256:488609bc63a29322326e05560731bf7bfea8e48ad646e1f5e40d366607de0942 \ - --hash=sha256:4d4987ce51a49370ea65c0bfd2234e8ce80a12780820d9dc462597a6e60d0841 +wheel==0.43.0 \ + --hash=sha256:465ef92c69fa5c5da2d1cf8ac40559a8c940886afcef87dcf14b9470862f1d85 \ + --hash=sha256:55c570405f142630c6b9f72fe09d9b67cf1477fcf543ae5b8dcb1f5b7377da81 # via -r requirements.in -zipp==3.17.0 \ - --hash=sha256:0e923e726174922dce09c53c59ad483ff7bbb8e572e00c7f7c46b88556409f31 \ - --hash=sha256:84e64a1c28cf7e91ed2078bb8cc8c259cb19b76942096c8d7b84947690cabaf0 +zipp==3.19.2 \ + --hash=sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19 \ + --hash=sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -setuptools==69.2.0 \ - --hash=sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e \ - --hash=sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c +setuptools==70.2.0 \ + --hash=sha256:b8b8060bb426838fbe942479c90296ce976249451118ef566a5a0b7d8b78fb05 \ + --hash=sha256:bd63e505105011b25c3c11f753f7e3b8465ea739efddaccef8f0efac2137bac1 # via -r requirements.in diff --git a/packages/pandas-gbq/.kokoro/test-samples-against-head.sh b/packages/pandas-gbq/.kokoro/test-samples-against-head.sh index 63ac41dfae1d..e9d8bd79a644 100755 --- a/packages/pandas-gbq/.kokoro/test-samples-against-head.sh +++ b/packages/pandas-gbq/.kokoro/test-samples-against-head.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/.kokoro/test-samples-impl.sh b/packages/pandas-gbq/.kokoro/test-samples-impl.sh index 5a0f5fab6a89..55910c8ba178 100755 --- a/packages/pandas-gbq/.kokoro/test-samples-impl.sh +++ b/packages/pandas-gbq/.kokoro/test-samples-impl.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/.kokoro/test-samples.sh b/packages/pandas-gbq/.kokoro/test-samples.sh index 50b35a48c190..7933d820149a 100755 --- a/packages/pandas-gbq/.kokoro/test-samples.sh +++ b/packages/pandas-gbq/.kokoro/test-samples.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/.kokoro/trampoline.sh b/packages/pandas-gbq/.kokoro/trampoline.sh index d85b1f267693..48f79699706e 100755 --- a/packages/pandas-gbq/.kokoro/trampoline.sh +++ b/packages/pandas-gbq/.kokoro/trampoline.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/.kokoro/trampoline_v2.sh b/packages/pandas-gbq/.kokoro/trampoline_v2.sh index 59a7cf3a9373..35fa529231dc 100755 --- a/packages/pandas-gbq/.kokoro/trampoline_v2.sh +++ b/packages/pandas-gbq/.kokoro/trampoline_v2.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/.pre-commit-config.yaml b/packages/pandas-gbq/.pre-commit-config.yaml index 6a8e16950664..1d74695f70b6 100644 --- a/packages/pandas-gbq/.pre-commit-config.yaml +++ b/packages/pandas-gbq/.pre-commit-config.yaml @@ -1,4 +1,4 @@ -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/.trampolinerc b/packages/pandas-gbq/.trampolinerc index a7dfeb42c6d0..0080152373d5 100644 --- a/packages/pandas-gbq/.trampolinerc +++ b/packages/pandas-gbq/.trampolinerc @@ -1,4 +1,4 @@ -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/MANIFEST.in b/packages/pandas-gbq/MANIFEST.in index e0a66705318e..d6814cd60037 100644 --- a/packages/pandas-gbq/MANIFEST.in +++ b/packages/pandas-gbq/MANIFEST.in @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/docs/conf.py b/packages/pandas-gbq/docs/conf.py index fe502297abf9..fbc405888214 100644 --- a/packages/pandas-gbq/docs/conf.py +++ b/packages/pandas-gbq/docs/conf.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/scripts/decrypt-secrets.sh b/packages/pandas-gbq/scripts/decrypt-secrets.sh index 0018b421ddf8..120b0ddc4364 100755 --- a/packages/pandas-gbq/scripts/decrypt-secrets.sh +++ b/packages/pandas-gbq/scripts/decrypt-secrets.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2023 Google LLC All rights reserved. +# Copyright 2024 Google LLC All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/pandas-gbq/scripts/readme-gen/readme_gen.py b/packages/pandas-gbq/scripts/readme-gen/readme_gen.py index 1acc119835b5..8f5e248a0da1 100644 --- a/packages/pandas-gbq/scripts/readme-gen/readme_gen.py +++ b/packages/pandas-gbq/scripts/readme-gen/readme_gen.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 99b7e675b2b314773f00decfc8843a176e21c93a Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Mon, 8 Jul 2024 16:28:53 -0700 Subject: [PATCH 422/519] chore(python): update dependencies in .kokoro (#792) Source-Link: https://github.com/googleapis/synthtool/commit/0142f3529bd44e1bd7297e72ac6d0c8228bf1489 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:99ab465187b4891e878ee4f9977b4a6aeeb0ceadf404870c416c50e06500eb42 Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 +- .../pandas-gbq/.kokoro/docker/docs/Dockerfile | 2 +- .../.kokoro/docker/docs/requirements.txt | 40 ++++++++-------- packages/pandas-gbq/.kokoro/requirements.txt | 46 +++++++++---------- 4 files changed, 48 insertions(+), 44 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 91d742b5b9fe..f9451fda6a80 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:d3de8a02819f65001effcbd3ea76ce97e9bcff035c7a89457f40f892c87c5b32 -# created: 2024-07-03T17:43:00.77142528Z + digest: sha256:99ab465187b4891e878ee4f9977b4a6aeeb0ceadf404870c416c50e06500eb42 +# created: 2024-07-08T16:17:14.833595692Z diff --git a/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile b/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile index a26ce61930f5..28de550f854c 100644 --- a/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile +++ b/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ubuntu:22.04 +from ubuntu:24.04 ENV DEBIAN_FRONTEND noninteractive diff --git a/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt b/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt index 0e5d70f20f83..7129c7715594 100644 --- a/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt +++ b/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt @@ -4,9 +4,9 @@ # # pip-compile --allow-unsafe --generate-hashes requirements.in # -argcomplete==3.2.3 \ - --hash=sha256:bf7900329262e481be5a15f56f19736b376df6f82ed27576fa893652c5de6c23 \ - --hash=sha256:c12355e0494c76a2a7b73e3a59b09024ca0ba1e279fb9ed6c1b82d5b74b6a70c +argcomplete==3.4.0 \ + --hash=sha256:69a79e083a716173e5532e0fa3bef45f793f4e61096cf52b5a42c0211c8b8aa5 \ + --hash=sha256:c2abcdfe1be8ace47ba777d4fce319eb13bf8ad9dace8d085dcad6eded88057f # via nox colorlog==6.8.2 \ --hash=sha256:3e3e079a41feb5a1b64f978b5ea4f46040a94f11f0e8bbb8261e3dbbeca64d44 \ @@ -16,23 +16,27 @@ distlib==0.3.8 \ --hash=sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784 \ --hash=sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64 # via virtualenv -filelock==3.13.1 \ - --hash=sha256:521f5f56c50f8426f5e03ad3b281b490a87ef15bc6c526f168290f0c7148d44e \ - --hash=sha256:57dbda9b35157b05fb3e58ee91448612eb674172fab98ee235ccb0b5bee19a1c +filelock==3.15.4 \ + --hash=sha256:2207938cbc1844345cb01a5a95524dae30f0ce089eba5b00378295a17e3e90cb \ + --hash=sha256:6ca1fffae96225dab4c6eaf1c4f4f28cd2568d3ec2a44e15a08520504de468e7 # via virtualenv -nox==2024.3.2 \ - --hash=sha256:e53514173ac0b98dd47585096a55572fe504fecede58ced708979184d05440be \ - --hash=sha256:f521ae08a15adbf5e11f16cb34e8d0e6ea521e0b92868f684e91677deb974553 +nox==2024.4.15 \ + --hash=sha256:6492236efa15a460ecb98e7b67562a28b70da006ab0be164e8821177577c0565 \ + --hash=sha256:ecf6700199cdfa9e5ea0a41ff5e6ef4641d09508eda6edb89d9987864115817f # via -r requirements.in -packaging==24.0 \ - --hash=sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5 \ - --hash=sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9 +packaging==24.1 \ + --hash=sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002 \ + --hash=sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124 # via nox -platformdirs==4.2.0 \ - --hash=sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068 \ - --hash=sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768 +platformdirs==4.2.2 \ + --hash=sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee \ + --hash=sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3 # via virtualenv -virtualenv==20.25.1 \ - --hash=sha256:961c026ac520bac5f69acb8ea063e8a4f071bcc9457b9c1f28f6b085c511583a \ - --hash=sha256:e08e13ecdca7a0bd53798f356d5831434afa5b07b93f0abdf0797b7a06ffe197 +tomli==2.0.1 \ + --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \ + --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f + # via nox +virtualenv==20.26.3 \ + --hash=sha256:4c43a2a236279d9ea36a0d76f98d84bd6ca94ac4e0f4a3b9d46d05e10fea542a \ + --hash=sha256:8cc4a31139e796e9a7de2cd5cf2489de1217193116a8fd42328f1bd65f434589 # via nox diff --git a/packages/pandas-gbq/.kokoro/requirements.txt b/packages/pandas-gbq/.kokoro/requirements.txt index 35ece0e4d2e9..9622baf0ba38 100644 --- a/packages/pandas-gbq/.kokoro/requirements.txt +++ b/packages/pandas-gbq/.kokoro/requirements.txt @@ -20,9 +20,9 @@ cachetools==5.3.3 \ --hash=sha256:0abad1021d3f8325b2fc1d2e9c8b9c9d57b04c3932657a72465447332c24d945 \ --hash=sha256:ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105 # via google-auth -certifi==2024.6.2 \ - --hash=sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516 \ - --hash=sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56 +certifi==2024.7.4 \ + --hash=sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b \ + --hash=sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90 # via requests cffi==1.16.0 \ --hash=sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc \ @@ -371,23 +371,23 @@ more-itertools==10.3.0 \ # via # jaraco-classes # jaraco-functools -nh3==0.2.17 \ - --hash=sha256:0316c25b76289cf23be6b66c77d3608a4fdf537b35426280032f432f14291b9a \ - --hash=sha256:1a814dd7bba1cb0aba5bcb9bebcc88fd801b63e21e2450ae6c52d3b3336bc911 \ - --hash=sha256:1aa52a7def528297f256de0844e8dd680ee279e79583c76d6fa73a978186ddfb \ - --hash=sha256:22c26e20acbb253a5bdd33d432a326d18508a910e4dcf9a3316179860d53345a \ - --hash=sha256:40015514022af31975c0b3bca4014634fa13cb5dc4dbcbc00570acc781316dcc \ - --hash=sha256:40d0741a19c3d645e54efba71cb0d8c475b59135c1e3c580f879ad5514cbf028 \ - --hash=sha256:551672fd71d06cd828e282abdb810d1be24e1abb7ae2543a8fa36a71c1006fe9 \ - --hash=sha256:66f17d78826096291bd264f260213d2b3905e3c7fae6dfc5337d49429f1dc9f3 \ - --hash=sha256:85cdbcca8ef10733bd31f931956f7fbb85145a4d11ab9e6742bbf44d88b7e351 \ - --hash=sha256:a3f55fabe29164ba6026b5ad5c3151c314d136fd67415a17660b4aaddacf1b10 \ - --hash=sha256:b4427ef0d2dfdec10b641ed0bdaf17957eb625b2ec0ea9329b3d28806c153d71 \ - --hash=sha256:ba73a2f8d3a1b966e9cdba7b211779ad8a2561d2dba9674b8a19ed817923f65f \ - --hash=sha256:c21bac1a7245cbd88c0b0e4a420221b7bfa838a2814ee5bb924e9c2f10a1120b \ - --hash=sha256:c551eb2a3876e8ff2ac63dff1585236ed5dfec5ffd82216a7a174f7c5082a78a \ - --hash=sha256:c790769152308421283679a142dbdb3d1c46c79c823008ecea8e8141db1a2062 \ - --hash=sha256:d7a25fd8c86657f5d9d576268e3b3767c5cd4f42867c9383618be8517f0f022a +nh3==0.2.18 \ + --hash=sha256:0411beb0589eacb6734f28d5497ca2ed379eafab8ad8c84b31bb5c34072b7164 \ + --hash=sha256:14c5a72e9fe82aea5fe3072116ad4661af5cf8e8ff8fc5ad3450f123e4925e86 \ + --hash=sha256:19aaba96e0f795bd0a6c56291495ff59364f4300d4a39b29a0abc9cb3774a84b \ + --hash=sha256:34c03fa78e328c691f982b7c03d4423bdfd7da69cd707fe572f544cf74ac23ad \ + --hash=sha256:36c95d4b70530b320b365659bb5034341316e6a9b30f0b25fa9c9eff4c27a204 \ + --hash=sha256:3a157ab149e591bb638a55c8c6bcb8cdb559c8b12c13a8affaba6cedfe51713a \ + --hash=sha256:42c64511469005058cd17cc1537578eac40ae9f7200bedcfd1fc1a05f4f8c200 \ + --hash=sha256:5f36b271dae35c465ef5e9090e1fdaba4a60a56f0bb0ba03e0932a66f28b9189 \ + --hash=sha256:6955369e4d9f48f41e3f238a9e60f9410645db7e07435e62c6a9ea6135a4907f \ + --hash=sha256:7b7c2a3c9eb1a827d42539aa64091640bd275b81e097cd1d8d82ef91ffa2e811 \ + --hash=sha256:8ce0f819d2f1933953fca255db2471ad58184a60508f03e6285e5114b6254844 \ + --hash=sha256:94a166927e53972a9698af9542ace4e38b9de50c34352b962f4d9a7d4c927af4 \ + --hash=sha256:a7f1b5b2c15866f2db413a3649a8fe4fd7b428ae58be2c0f6bca5eefd53ca2be \ + --hash=sha256:c8b3a1cebcba9b3669ed1a84cc65bf005728d2f0bc1ed2a6594a992e817f3a50 \ + --hash=sha256:de3ceed6e661954871d6cd78b410213bdcb136f79aafe22aa7182e028b8c7307 \ + --hash=sha256:f0eca9ca8628dbb4e916ae2491d72957fdd35f7a5d326b7032a345f111ac07fe # via readme-renderer nox==2024.4.15 \ --hash=sha256:6492236efa15a460ecb98e7b67562a28b70da006ab0be164e8821177577c0565 \ @@ -460,9 +460,9 @@ python-dateutil==2.9.0.post0 \ --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 # via gcp-releasetool -readme-renderer==43.0 \ - --hash=sha256:1818dd28140813509eeed8d62687f7cd4f7bad90d4db586001c5dc09d4fde311 \ - --hash=sha256:19db308d86ecd60e5affa3b2a98f017af384678c63c88e5d4556a380e674f3f9 +readme-renderer==44.0 \ + --hash=sha256:2fbca89b81a08526aadf1357a8c2ae889ec05fb03f5da67f9769c9a592166151 \ + --hash=sha256:8712034eabbfa6805cacf1402b4eeb2a73028f72d1166d6f5cb7f9c047c5d1e1 # via twine requests==2.32.3 \ --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \ From 0f782a39ddbccbce67e64b49b6d4b0c4695636db Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 9 Jul 2024 15:44:17 -0400 Subject: [PATCH 423/519] chore(python): use python 3.10 for docs build (#793) * chore(python): use python 3.10 for docs build Source-Link: https://github.com/googleapis/synthtool/commit/9ae07858520bf035a3d5be569b5a65d960ee4392 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:52210e0e0559f5ea8c52be148b33504022e1faef4e95fbe4b32d68022af2fa7e * use python 3.10 for docs build --------- Co-authored-by: Owl Bot Co-authored-by: Lingqing Gan Co-authored-by: Anthonios Partheniou --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 ++-- .../pandas-gbq/.kokoro/docker/docs/Dockerfile | 19 +++++++++++-------- packages/pandas-gbq/noxfile.py | 5 ++++- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index f9451fda6a80..f30cb3775afc 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:99ab465187b4891e878ee4f9977b4a6aeeb0ceadf404870c416c50e06500eb42 -# created: 2024-07-08T16:17:14.833595692Z + digest: sha256:52210e0e0559f5ea8c52be148b33504022e1faef4e95fbe4b32d68022af2fa7e +# created: 2024-07-08T19:25:35.862283192Z diff --git a/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile b/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile index 28de550f854c..5205308b334d 100644 --- a/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile +++ b/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile @@ -40,7 +40,6 @@ RUN apt-get update \ libssl-dev \ libsqlite3-dev \ portaudio19-dev \ - python3-distutils \ redis-server \ software-properties-common \ ssh \ @@ -60,18 +59,22 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ && rm -f /var/cache/apt/archives/*.deb -###################### Install python 3.9.13 -# Download python 3.9.13 -RUN wget https://www.python.org/ftp/python/3.9.13/Python-3.9.13.tgz +###################### Install python 3.10.14 for docs/docfx session + +# Download python 3.10.14 +RUN wget https://www.python.org/ftp/python/3.10.14/Python-3.10.14.tgz # Extract files -RUN tar -xvf Python-3.9.13.tgz +RUN tar -xvf Python-3.10.14.tgz -# Install python 3.9.13 -RUN ./Python-3.9.13/configure --enable-optimizations +# Install python 3.10.14 +RUN ./Python-3.10.14/configure --enable-optimizations RUN make altinstall +RUN python3.10 -m venv /venv +ENV PATH /venv/bin:$PATH + ###################### Install pip RUN wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \ && python3 /tmp/get-pip.py \ @@ -84,4 +87,4 @@ RUN python3 -m pip COPY requirements.txt /requirements.txt RUN python3 -m pip install --require-hashes -r requirements.txt -CMD ["python3.8"] +CMD ["python3.10"] diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 52590f366f93..a8c12aa1d492 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -78,6 +78,9 @@ CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() +# Error if a python version is missing +nox.options.error_on_missing_interpreters = True + def _calculate_duration(func): """This decorator prints the execution time for the decorated function.""" @@ -420,7 +423,7 @@ def cover(session): session.run("coverage", "erase") -@nox.session(python="3.9") +@nox.session(python="3.10") @_calculate_duration def docs(session): """Build the docs for this library.""" From 96dd394c83f90cbe6c1c2f7c9d2f5e9fe4640cb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Swe=C3=B1a=20=28Swast=29?= Date: Tue, 9 Jul 2024 15:54:18 -0500 Subject: [PATCH 424/519] chore: remove references to conda (#795) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See also, internal change 650726157. Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - [ ] Make sure to open an issue as a [bug/issue](https://togithub.com/googleapis/python-bigquery-pandas/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea - [ ] Ensure the tests and linter pass - [ ] Code coverage does not decrease (if any source code was changed) - [ ] Appropriate docs were updated (if necessary) 🦕 --- packages/pandas-gbq/.kokoro/build.sh | 16 +-- .../.kokoro/presubmit/conda_test.cfg | 7 -- packages/pandas-gbq/CHANGELOG.md | 3 - packages/pandas-gbq/README.rst | 14 +-- packages/pandas-gbq/docs/contributing.rst | 22 ----- packages/pandas-gbq/docs/install.rst | 10 +- packages/pandas-gbq/docs/reading.rst | 5 - packages/pandas-gbq/noxfile.py | 99 +------------------ packages/pandas-gbq/owlbot.py | 6 +- packages/pandas-gbq/release-procedure.md | 9 +- 10 files changed, 10 insertions(+), 181 deletions(-) delete mode 100644 packages/pandas-gbq/.kokoro/presubmit/conda_test.cfg diff --git a/packages/pandas-gbq/.kokoro/build.sh b/packages/pandas-gbq/.kokoro/build.sh index e490fe53f95a..08171cbd47a9 100755 --- a/packages/pandas-gbq/.kokoro/build.sh +++ b/packages/pandas-gbq/.kokoro/build.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,18 +23,9 @@ cd "${PROJECT_ROOT}" # Disable buffering, so that the logs stream through. export PYTHONUNBUFFERED=1 -export CONDA_EXE=/root/conda/bin/conda -export CONDA_PREFIX=/root/conda -export CONDA_PROMPT_MODIFIER=(base) -export _CE_CONDA= -export CONDA_SHLVL=1 -export CONDA_PYTHON_EXE=/root/conda/bin/python -export CONDA_DEFAULT_ENV=base -export PATH=/root/conda/bin:/root/conda/condabin:${PATH} - # Debug: show build environment -env +env | grep KOKORO # Setup service account credentials. export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json @@ -42,9 +33,6 @@ export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json # Setup project id. export PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json") -# Install nox -python3 -m pip install --upgrade --quiet nox - # If this is a continuous build, send the test log to the FlakyBot. # See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot. if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"continuous"* ]]; then diff --git a/packages/pandas-gbq/.kokoro/presubmit/conda_test.cfg b/packages/pandas-gbq/.kokoro/presubmit/conda_test.cfg deleted file mode 100644 index 6e3943f35021..000000000000 --- a/packages/pandas-gbq/.kokoro/presubmit/conda_test.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Only run this nox session. -env_vars: { - key: "NOX_SESSION" - value: "conda_test" -} diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index 418201bfa915..ced3da531fb8 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -6,7 +6,6 @@ ### Bug Fixes * Handle None when converting numerics to parquet ([#768](https://github.com/googleapis/python-bigquery-pandas/issues/768)) ([53a4683](https://github.com/googleapis/python-bigquery-pandas/commit/53a46833a320963d5c15427f6eb631e0199fb332)) -* Set minimum allowable version of sqlite when performing a conda install ([#780](https://github.com/googleapis/python-bigquery-pandas/issues/780)) ([8a03d44](https://github.com/googleapis/python-bigquery-pandas/commit/8a03d44fbe125ae1202f43b7c6e54c98eca94d4d)) ### Documentation @@ -369,8 +368,6 @@ df = gbq.read_gbq( ([#281](https://github.com/googleapis/python-bigquery-pandas/issues/281)) - Fix `pytest.raises` usage for latest pytest. Fix warnings in tests. ([#282](https://github.com/googleapis/python-bigquery-pandas/issues/282)) -- Update CI to install nightly packages in the conda tests. - ([#254](https://github.com/googleapis/python-bigquery-pandas/issues/254)) ## 0.10.0 / 2019-04-05 diff --git a/packages/pandas-gbq/README.rst b/packages/pandas-gbq/README.rst index 1b35f26a91eb..122f43ee09e8 100644 --- a/packages/pandas-gbq/README.rst +++ b/packages/pandas-gbq/README.rst @@ -1,7 +1,7 @@ pandas-gbq ========== -|preview| |pypi| |versions| +|preview| |pypi| |versions| **pandas-gbq** is a package providing an interface to the Google BigQuery API from pandas. @@ -20,14 +20,6 @@ pandas-gbq Installation ------------ - -Install latest release version via conda -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: shell - - $ conda install pandas-gbq --channel conda-forge - Install latest release version via pip ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -52,7 +44,7 @@ Perform a query .. code:: python import pandas_gbq - + result_dataframe = pandas_gbq.read_gbq("SELECT column FROM dataset.table WHERE value = 'something'") Upload a dataframe @@ -61,7 +53,7 @@ Upload a dataframe .. code:: python import pandas_gbq - + pandas_gbq.to_gbq(dataframe, "dataset.table") More samples diff --git a/packages/pandas-gbq/docs/contributing.rst b/packages/pandas-gbq/docs/contributing.rst index 891a1799edc2..39c2694a623b 100644 --- a/packages/pandas-gbq/docs/contributing.rst +++ b/packages/pandas-gbq/docs/contributing.rst @@ -145,31 +145,9 @@ Install in Development Mode It's helpful to install pandas-gbq in development mode so that you can use the library without reinstalling the package after every change. -Conda -~~~~~ - -Create a new conda environment and install the necessary dependencies - -.. code-block:: shell - - $ conda create -n my-env --channel conda-forge \ - db-dtypes \ - pandas \ - pydata-google-auth \ - google-cloud-bigquery - $ source activate my-env - -Install pandas-gbq in development mode - -.. code-block:: shell - - $ python setup.py develop - Pip & virtualenv ~~~~~~~~~~~~~~~~ -*Skip this section if you already followed the conda instructions.* - Create a new `virtual environment `__. diff --git a/packages/pandas-gbq/docs/install.rst b/packages/pandas-gbq/docs/install.rst index 9887c79962ed..849b1b60f118 100644 --- a/packages/pandas-gbq/docs/install.rst +++ b/packages/pandas-gbq/docs/install.rst @@ -1,16 +1,8 @@ Installation ============ -You can install pandas-gbq with ``conda``, ``pip``, or by installing from source. +You can install pandas-gbq with ``pip`` or by installing from source. -Conda ------ - -.. code-block:: shell - - $ conda install pandas-gbq --channel conda-forge - -This installs pandas-gbq and all common dependencies, including ``pandas``. Pip --- diff --git a/packages/pandas-gbq/docs/reading.rst b/packages/pandas-gbq/docs/reading.rst index bc7b74e150b3..5fa369a7c7b4 100644 --- a/packages/pandas-gbq/docs/reading.rst +++ b/packages/pandas-gbq/docs/reading.rst @@ -103,11 +103,6 @@ quickly (but at an `increased cost pip install --upgrade google-cloud-bigquery-storage pyarrow - With conda: - - .. code-block:: sh - - conda install -c conda-forge google-cloud-bigquery-storage #. Set ``use_bqstorage_api`` to ``True`` when calling the :func:`~pandas_gbq.read_gbq` function. As of the ``google-cloud-bigquery`` package, version 1.11.1 or later,the function will fallback to the diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index a8c12aa1d492..774325aebc94 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -23,7 +23,6 @@ import pathlib import re import shutil -import subprocess import time import warnings @@ -57,11 +56,6 @@ "3.9": [], } -CONDA_TEST_PYTHON_VERSIONS = [ - UNIT_TEST_PYTHON_VERSIONS[0], - UNIT_TEST_PYTHON_VERSIONS[-1], -] - SYSTEM_TEST_PYTHON_VERSIONS = ["3.8", "3.9", "3.10", "3.11", "3.12"] SYSTEM_TEST_STANDARD_DEPENDENCIES = [ "mock", @@ -316,6 +310,7 @@ def system(session): @_calculate_duration def prerelease(session): session.install( + # https://arrow.apache.org/docs/developers/python.html#installing-nightly-packages "--extra-index-url", "https://pypi.fury.io/arrow-nightlies/", "--prefer-binary", @@ -323,14 +318,6 @@ def prerelease(session): "--upgrade", "pyarrow", ) - session.install( - "--extra-index-url", - "https://pypi.anaconda.org/scipy-wheels-nightly/simple", - "--prefer-binary", - "--pre", - "--upgrade", - "pandas", - ) session.install( "--prefer-binary", "--pre", @@ -342,6 +329,7 @@ def prerelease(session): "google-resumable-media", # Exclude version 1.49.0rc1 which has a known issue. See https://github.com/grpc/grpc/pull/30642 "grpcio!=1.49.0rc1", + "pandas", ) session.install( "freezegun", @@ -504,86 +492,3 @@ def docfx(session): os.path.join("docs", ""), os.path.join("docs", "_build", "html", ""), ) - - -def install_conda_unittest_dependencies(session, standard_deps, conda_forge_packages): - """Installs packages from conda forge, pypi, and locally.""" - - # Install from conda-forge and default conda package repos. - session.conda_install(*conda_forge_packages, channel=["defaults", "conda-forge"]) - - # Install from pypi for packages not readily available on conda forge. - session.install( - *standard_deps, - ) - - # Install via pip from the local repo, avoid doing dependency resolution - # via pip, so that we don't override any conda resolved dependencies - session.install("-e", ".", "--no-deps") - - -@nox.session(python=CONDA_TEST_PYTHON_VERSIONS, venv_backend="mamba") -@_calculate_duration -def conda_test(session): - """Run test suite in a conda virtual environment. - - Installs all test dependencies, then installs this package. - NOTE: Some of these libraries are not readily available on conda-forge - at this time and are thus installed using pip after the base install of - libraries from conda-forge. - - We decided that it was more important to prove a base ability to install - using conda than to complicate things with adding a whole nother - set of constraints just for a conda install, so this install does not - attempt to constrain packages (i.e. in a constraints-x.x.txt file) - manually. - """ - - standard_deps = ( - UNIT_TEST_STANDARD_DEPENDENCIES - + UNIT_TEST_DEPENDENCIES - + UNIT_TEST_EXTERNAL_DEPENDENCIES - ) - - conda_forge_packages = [ - "db-dtypes", - "google-api-core", - "google-auth", - "google-auth-oauthlib", - "google-cloud-bigquery", - "google-cloud-bigquery-storage", - "numpy", - "pandas", - "pyarrow", - "pydata-google-auth", - "tqdm", - "protobuf", - "sqlite>3.31.1", # v3.31.1 caused test failures - ] - - install_conda_unittest_dependencies(session, standard_deps, conda_forge_packages) - - # Provide a list of all installed packages (both from conda forge and pip) - # for troubleshooting purposes. - session.run("mamba", "list") - - # Using subprocess.run() instead of session.run() because - # session.run() does not correctly handle the pip check command. - subprocess.run( - ["pip", "check"], check=True - ) # Raise an exception if pip check fails - - # Tests are limited to unit tests only, at this time. - session.run( - "py.test", - "--quiet", - f"--junitxml=unit_{session.python}_sponge_log.xml", - "--cov=pandas_gbq", - "--cov=tests/unit", - "--cov-append", - "--cov-config=.coveragerc", - "--cov-report=", - "--cov-fail-under=0", - os.path.join("tests", "unit"), - *session.posargs, - ) diff --git a/packages/pandas-gbq/owlbot.py b/packages/pandas-gbq/owlbot.py index 3bb53fc2d16a..aeda7356bb0f 100644 --- a/packages/pandas-gbq/owlbot.py +++ b/packages/pandas-gbq/owlbot.py @@ -56,13 +56,9 @@ "docs/multiprocessing.rst", "noxfile.py", "README.rst", - - # exclude .kokoro/build.sh which is customized due to support for conda - ".kokoro/build.sh", - # exclude this file as we have an alternate prerelease.cfg ".kokoro/presubmit/prerelease-deps.cfg", - ] + ], ) # ---------------------------------------------------------------------------- diff --git a/packages/pandas-gbq/release-procedure.md b/packages/pandas-gbq/release-procedure.md index 3b33021d5d07..47c8fd74fe7a 100644 --- a/packages/pandas-gbq/release-procedure.md +++ b/packages/pandas-gbq/release-procedure.md @@ -33,13 +33,6 @@ twine upload dist/* -* Create the [release on GitHub](https://github.com/pydata/pandas-gbq/releases/new) using the tag created earlier. +* Create the [release on GitHub](https://github.com/googleapis/python-bigquery-pandas/releases/new) using the tag created earlier. * Upload wheel and source zip from `dist/` directory. - -* Do a pull-request to the feedstock on `pandas-gbq-feedstock `__ - (Or review PR from @regro-cf-autotick-bot which updates the feedstock). - - * update the version - * update the SHA256 (retrieve from PyPI) - * update the dependencies (if they changed) From 72f5f80c8a56fe531adccc46209ea38c5c308624 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Tue, 16 Jul 2024 15:32:18 -0400 Subject: [PATCH 425/519] test: adds a config to isolate the system- CI/CD tests (#796) Adds a config to isolate the system-3.8 and system-3.12 CI/CD tests from the presubmit job into two separate jobs to increase parallelism. --- packages/pandas-gbq/.kokoro/presubmit/system-3.12.cfg | 7 +++++++ packages/pandas-gbq/.kokoro/presubmit/system-3.8.cfg | 7 +++++++ 2 files changed, 14 insertions(+) create mode 100644 packages/pandas-gbq/.kokoro/presubmit/system-3.12.cfg create mode 100644 packages/pandas-gbq/.kokoro/presubmit/system-3.8.cfg diff --git a/packages/pandas-gbq/.kokoro/presubmit/system-3.12.cfg b/packages/pandas-gbq/.kokoro/presubmit/system-3.12.cfg new file mode 100644 index 000000000000..28bbbe4c25b7 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/presubmit/system-3.12.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run the following session(s) +env_vars: { + key: "NOX_SESSION" + value: "system-3.12" +} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/presubmit/system-3.8.cfg b/packages/pandas-gbq/.kokoro/presubmit/system-3.8.cfg new file mode 100644 index 000000000000..15b14528baf7 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/presubmit/system-3.8.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run the following session(s) +env_vars: { + key: "NOX_SESSION" + value: "system-3.8" +} \ No newline at end of file From 0815626cc8b16d9eee0053de141578a85d518c4e Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Mon, 22 Jul 2024 15:47:17 -0400 Subject: [PATCH 426/519] test: update the ci/cd pipeline (#794) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test: update the ci/cd pipeline * remove reference to cover and clean up linting error * Update noxfile.py * define sessions that run during presubmit * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * updates owlbot to avoid overwriting presubmit.cfg * restores file changes that owlbot took out * updates session name --------- Co-authored-by: Owl Bot --- packages/pandas-gbq/.kokoro/presubmit/presubmit.cfg | 8 +++++++- packages/pandas-gbq/noxfile.py | 3 --- packages/pandas-gbq/owlbot.py | 1 + 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/pandas-gbq/.kokoro/presubmit/presubmit.cfg b/packages/pandas-gbq/.kokoro/presubmit/presubmit.cfg index 8f43917d92fe..f4651182a0de 100644 --- a/packages/pandas-gbq/.kokoro/presubmit/presubmit.cfg +++ b/packages/pandas-gbq/.kokoro/presubmit/presubmit.cfg @@ -1 +1,7 @@ -# Format: //devtools/kokoro/config/proto/build.proto \ No newline at end of file +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run the following session(s) +env_vars: { + key: "NOX_SESSION" + value: "blacken lint lint_setup_py docs" +} \ No newline at end of file diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 774325aebc94..d316dac8dc19 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -98,9 +98,6 @@ def wrapper(*args, **kwargs): # 'docfx' is excluded since it only needs to run in 'docs-presubmit' nox.options.sessions = [ - "unit", - "system", - "cover", "lint", "lint_setup_py", "blacken", diff --git a/packages/pandas-gbq/owlbot.py b/packages/pandas-gbq/owlbot.py index aeda7356bb0f..916a7074f1bd 100644 --- a/packages/pandas-gbq/owlbot.py +++ b/packages/pandas-gbq/owlbot.py @@ -58,6 +58,7 @@ "README.rst", # exclude this file as we have an alternate prerelease.cfg ".kokoro/presubmit/prerelease-deps.cfg", + ".kokoro/presubmit/presubmit.cfg", ], ) From 27adc268e60ff058fc22856d54d2772b1350080b Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Tue, 23 Jul 2024 22:23:51 +0200 Subject: [PATCH 427/519] chore(deps): update dependency pyarrow to v17 (#797) Co-authored-by: Chalmer Lowe Co-authored-by: Lingqing Gan --- packages/pandas-gbq/samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index cef89843b8c5..f05754448706 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -3,4 +3,4 @@ google-cloud-bigquery==3.25.0 pandas-gbq==0.23.1 pandas===2.0.3; python_version == '3.8' pandas==2.2.2; python_version >= '3.9' -pyarrow==16.1.0; python_version >= '3.8' +pyarrow==17.0.0; python_version >= '3.8' From 5a746742c7d78298fff8448eb268004598042cd3 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Wed, 24 Jul 2024 15:42:13 +0200 Subject: [PATCH 428/519] chore(deps): update dependency pytest to v8.3.1 (#798) --- packages/pandas-gbq/samples/snippets/requirements-test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements-test.txt b/packages/pandas-gbq/samples/snippets/requirements-test.txt index 7af5d0fecadb..cb1bc07ee6e9 100644 --- a/packages/pandas-gbq/samples/snippets/requirements-test.txt +++ b/packages/pandas-gbq/samples/snippets/requirements-test.txt @@ -1,2 +1,2 @@ google-cloud-testutils==1.4.0 -pytest==8.2.2 +pytest==8.3.1 From 7b8462b2d2b44153ce44edf39cf74253efa8b184 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Mon, 29 Jul 2024 22:31:04 +0200 Subject: [PATCH 429/519] chore(deps): update dependency pytest to v8.3.2 (#800) --- packages/pandas-gbq/samples/snippets/requirements-test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements-test.txt b/packages/pandas-gbq/samples/snippets/requirements-test.txt index cb1bc07ee6e9..aedea97ec04c 100644 --- a/packages/pandas-gbq/samples/snippets/requirements-test.txt +++ b/packages/pandas-gbq/samples/snippets/requirements-test.txt @@ -1,2 +1,2 @@ google-cloud-testutils==1.4.0 -pytest==8.3.1 +pytest==8.3.2 From f5a85a1b3fc4b3a067bffcb9eabfbf5680a82393 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 15 Aug 2024 11:25:27 -0700 Subject: [PATCH 430/519] chore(python): fix docs build (#801) Source-Link: https://github.com/googleapis/synthtool/commit/bef813d194de29ddf3576eda60148b6b3dcc93d9 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:94bb690db96e6242b2567a4860a94d48fa48696d092e51b0884a1a2c0a79a407 Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 ++-- .../pandas-gbq/.kokoro/docker/docs/Dockerfile | 9 ++++----- packages/pandas-gbq/.kokoro/publish-docs.sh | 20 +++++++++---------- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index f30cb3775afc..6d064ddb9b06 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:52210e0e0559f5ea8c52be148b33504022e1faef4e95fbe4b32d68022af2fa7e -# created: 2024-07-08T19:25:35.862283192Z + digest: sha256:94bb690db96e6242b2567a4860a94d48fa48696d092e51b0884a1a2c0a79a407 +# created: 2024-07-31T14:52:44.926548819Z diff --git a/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile b/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile index 5205308b334d..e5410e296bd8 100644 --- a/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile +++ b/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile @@ -72,19 +72,18 @@ RUN tar -xvf Python-3.10.14.tgz RUN ./Python-3.10.14/configure --enable-optimizations RUN make altinstall -RUN python3.10 -m venv /venv -ENV PATH /venv/bin:$PATH +ENV PATH /usr/local/bin/python3.10:$PATH ###################### Install pip RUN wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \ - && python3 /tmp/get-pip.py \ + && python3.10 /tmp/get-pip.py \ && rm /tmp/get-pip.py # Test pip -RUN python3 -m pip +RUN python3.10 -m pip # Install build requirements COPY requirements.txt /requirements.txt -RUN python3 -m pip install --require-hashes -r requirements.txt +RUN python3.10 -m pip install --require-hashes -r requirements.txt CMD ["python3.10"] diff --git a/packages/pandas-gbq/.kokoro/publish-docs.sh b/packages/pandas-gbq/.kokoro/publish-docs.sh index 38f083f05aa0..233205d580e9 100755 --- a/packages/pandas-gbq/.kokoro/publish-docs.sh +++ b/packages/pandas-gbq/.kokoro/publish-docs.sh @@ -21,18 +21,18 @@ export PYTHONUNBUFFERED=1 export PATH="${HOME}/.local/bin:${PATH}" # Install nox -python3 -m pip install --require-hashes -r .kokoro/requirements.txt -python3 -m nox --version +python3.10 -m pip install --require-hashes -r .kokoro/requirements.txt +python3.10 -m nox --version # build docs nox -s docs # create metadata -python3 -m docuploader create-metadata \ +python3.10 -m docuploader create-metadata \ --name=$(jq --raw-output '.name // empty' .repo-metadata.json) \ - --version=$(python3 setup.py --version) \ + --version=$(python3.10 setup.py --version) \ --language=$(jq --raw-output '.language // empty' .repo-metadata.json) \ - --distribution-name=$(python3 setup.py --name) \ + --distribution-name=$(python3.10 setup.py --name) \ --product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \ --github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \ --issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json) @@ -40,18 +40,18 @@ python3 -m docuploader create-metadata \ cat docs.metadata # upload docs -python3 -m docuploader upload docs/_build/html --metadata-file docs.metadata --staging-bucket "${STAGING_BUCKET}" +python3.10 -m docuploader upload docs/_build/html --metadata-file docs.metadata --staging-bucket "${STAGING_BUCKET}" # docfx yaml files nox -s docfx # create metadata. -python3 -m docuploader create-metadata \ +python3.10 -m docuploader create-metadata \ --name=$(jq --raw-output '.name // empty' .repo-metadata.json) \ - --version=$(python3 setup.py --version) \ + --version=$(python3.10 setup.py --version) \ --language=$(jq --raw-output '.language // empty' .repo-metadata.json) \ - --distribution-name=$(python3 setup.py --name) \ + --distribution-name=$(python3.10 setup.py --name) \ --product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \ --github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \ --issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json) @@ -59,4 +59,4 @@ python3 -m docuploader create-metadata \ cat docs.metadata # upload docs -python3 -m docuploader upload docs/_build/html/docfx_yaml --metadata-file docs.metadata --destination-prefix docfx --staging-bucket "${V2_STAGING_BUCKET}" +python3.10 -m docuploader upload docs/_build/html/docfx_yaml --metadata-file docs.metadata --destination-prefix docfx --staging-bucket "${V2_STAGING_BUCKET}" From 7733294248ad24e853b925f7312aeaabd0be9d7c Mon Sep 17 00:00:00 2001 From: Carlos O'Ryan Date: Fri, 16 Aug 2024 18:10:51 +0000 Subject: [PATCH 431/519] docs: fix typo in 'vebosity' (#803) Co-authored-by: Lingqing Gan --- packages/pandas-gbq/docs/intro.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/docs/intro.rst b/packages/pandas-gbq/docs/intro.rst index ab93b4fe2ba9..3bb60ba77a06 100644 --- a/packages/pandas-gbq/docs/intro.rst +++ b/packages/pandas-gbq/docs/intro.rst @@ -36,8 +36,8 @@ download the results as a :class:`pandas.DataFrame` object. By default, queries use standard SQL syntax. Visit the :doc:`reading tables guide ` to learn about the available options. -Adjusting log vebosity -^^^^^^^^^^^^^^^^^^^^^^ +Adjusting log verbosity +^^^^^^^^^^^^^^^^^^^^^^^ Because some requests take some time, this library will log its progress of longer queries. IPython & Jupyter by default attach a handler to the logger. From 1e0a8e2dcdafa6d3d93af9ed361668bc37961246 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 10 Sep 2024 10:55:26 -0700 Subject: [PATCH 432/519] chore(python): update unittest workflow template (#808) Source-Link: https://github.com/googleapis/synthtool/commit/e6f91eb4db419b02af74197905b99fa00a6030c0 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:365d92ef2206cfad00a8c5955c36789d0de124e2b6d92a72dd0486315a0f2e57 Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 ++-- packages/pandas-gbq/.github/workflows/unittest.yml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 6d064ddb9b06..f8bd8149fa87 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:94bb690db96e6242b2567a4860a94d48fa48696d092e51b0884a1a2c0a79a407 -# created: 2024-07-31T14:52:44.926548819Z + digest: sha256:365d92ef2206cfad00a8c5955c36789d0de124e2b6d92a72dd0486315a0f2e57 +# created: 2024-09-04T14:50:52.658171431Z diff --git a/packages/pandas-gbq/.github/workflows/unittest.yml b/packages/pandas-gbq/.github/workflows/unittest.yml index 3c11914b035e..7be33c8f1b48 100644 --- a/packages/pandas-gbq/.github/workflows/unittest.yml +++ b/packages/pandas-gbq/.github/workflows/unittest.yml @@ -30,6 +30,7 @@ jobs: with: name: coverage-artifact-${{ matrix.python }} path: .coverage-${{ matrix.python }} + include-hidden-files: true cover: runs-on: ubuntu-latest From d77dbe1cb1a5c78b7dffd5b3e910a590236eba77 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 18 Sep 2024 11:42:41 -0400 Subject: [PATCH 433/519] fix(deps): require packaging >= 22.0 (#811) * fix(deps): require packaging >= 22.0 * require packaging >= 22.0 * fix(deps): require google-cloud-bigquery 3.4.2 * testing numpy * fix(deps): require numpy >=1.18.1 --- packages/pandas-gbq/pandas_gbq/features.py | 2 +- packages/pandas-gbq/setup.py | 6 +++--- packages/pandas-gbq/testing/constraints-3.8.txt | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/features.py b/packages/pandas-gbq/pandas_gbq/features.py index 2871d5eaa3cb..62405b4c1f9e 100644 --- a/packages/pandas-gbq/pandas_gbq/features.py +++ b/packages/pandas-gbq/pandas_gbq/features.py @@ -5,7 +5,7 @@ """Module for checking dependency versions and supported features.""" # https://github.com/googleapis/python-bigquery/blob/main/CHANGELOG.md -BIGQUERY_MINIMUM_VERSION = "3.3.5" +BIGQUERY_MINIMUM_VERSION = "3.4.2" BIGQUERY_QUERY_AND_WAIT_VERSION = "3.14.0" PANDAS_VERBOSITY_DEPRECATION_VERSION = "0.23.0" PANDAS_BOOLEAN_DTYPE_VERSION = "1.0.0" diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index 461cbfdee234..df793e59779c 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -23,7 +23,7 @@ dependencies = [ "setuptools", "db-dtypes >=1.0.4,<2.0.0", - "numpy >=1.16.6", + "numpy >=1.18.1", "pandas >=1.1.4", "pyarrow >=3.0.0", "pydata-google-auth >=1.5.0", @@ -35,8 +35,8 @@ "google-auth-oauthlib >=0.7.0", # Please also update the minimum version in pandas_gbq/features.py to # allow pandas-gbq to detect invalid package versions at runtime. - "google-cloud-bigquery >=3.3.5,<4.0.0dev", - "packaging >=20.0.0", + "google-cloud-bigquery >=3.4.2,<4.0.0dev", + "packaging >=22.0.0", ] extras = { "bqstorage": [ diff --git a/packages/pandas-gbq/testing/constraints-3.8.txt b/packages/pandas-gbq/testing/constraints-3.8.txt index f77e0f2d4140..e551d17e3e92 100644 --- a/packages/pandas-gbq/testing/constraints-3.8.txt +++ b/packages/pandas-gbq/testing/constraints-3.8.txt @@ -10,11 +10,11 @@ db-dtypes==1.0.4 google-api-core==2.10.2 google-auth==2.13.0 google-auth-oauthlib==0.7.0 -google-cloud-bigquery==3.3.5 +google-cloud-bigquery==3.4.2 google-cloud-bigquery-storage==2.16.2 -numpy==1.16.6 +numpy==1.18.1 pandas==1.1.4 pyarrow==3.0.0 pydata-google-auth==1.5.0 tqdm==4.23.0 -packaging==20.0.0 \ No newline at end of file +packaging==22.0.0 \ No newline at end of file From 000e957d4187a41dea6be33a87ffa34ac731f3b8 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Wed, 18 Sep 2024 08:53:27 -0700 Subject: [PATCH 434/519] build(python): release script update (#810) Source-Link: https://github.com/googleapis/synthtool/commit/71a72973dddbc66ea64073b53eda49f0d22e0942 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:e8dcfd7cbfd8beac3a3ff8d3f3185287ea0625d859168cc80faccfc9a7a00455 Co-authored-by: Owl Bot Co-authored-by: Anthonios Partheniou --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 ++-- packages/pandas-gbq/.kokoro/release.sh | 2 +- packages/pandas-gbq/.kokoro/release/common.cfg | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index f8bd8149fa87..597e0c3261ca 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:365d92ef2206cfad00a8c5955c36789d0de124e2b6d92a72dd0486315a0f2e57 -# created: 2024-09-04T14:50:52.658171431Z + digest: sha256:e8dcfd7cbfd8beac3a3ff8d3f3185287ea0625d859168cc80faccfc9a7a00455 +# created: 2024-09-16T21:04:09.091105552Z diff --git a/packages/pandas-gbq/.kokoro/release.sh b/packages/pandas-gbq/.kokoro/release.sh index 1902895d1fc7..f7527caa4e35 100755 --- a/packages/pandas-gbq/.kokoro/release.sh +++ b/packages/pandas-gbq/.kokoro/release.sh @@ -23,7 +23,7 @@ python3 -m releasetool publish-reporter-script > /tmp/publisher-script; source / export PYTHONUNBUFFERED=1 # Move into the package, build the distribution and upload. -TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google-cloud-pypi-token-keystore-1") +TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google-cloud-pypi-token-keystore-2") cd github/python-bigquery-pandas python3 setup.py sdist bdist_wheel twine upload --username __token__ --password "${TWINE_PASSWORD}" dist/* diff --git a/packages/pandas-gbq/.kokoro/release/common.cfg b/packages/pandas-gbq/.kokoro/release/common.cfg index 7b60d2107274..8ad3ddb2061e 100644 --- a/packages/pandas-gbq/.kokoro/release/common.cfg +++ b/packages/pandas-gbq/.kokoro/release/common.cfg @@ -28,7 +28,7 @@ before_action { fetch_keystore { keystore_resource { keystore_config_id: 73713 - keyname: "google-cloud-pypi-token-keystore-1" + keyname: "google-cloud-pypi-token-keystore-2" } } } From 2afe9651a66495f70363e0c7574af41fa4a5e46b Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Thu, 19 Sep 2024 00:20:15 +0200 Subject: [PATCH 435/519] chore(deps): update all dependencies (#809) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update all dependencies * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot Co-authored-by: Lingqing Gan Co-authored-by: Anthonios Partheniou --- packages/pandas-gbq/samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index f05754448706..47c271255cb9 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,4 +1,4 @@ -google-cloud-bigquery-storage==2.25.0 +google-cloud-bigquery-storage==2.26.0 google-cloud-bigquery==3.25.0 pandas-gbq==0.23.1 pandas===2.0.3; python_version == '3.8' From c1bc0d0e5d6f905a3385af4096408109517b73f7 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Fri, 20 Sep 2024 23:27:33 +0200 Subject: [PATCH 436/519] chore(deps): update all dependencies (#813) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update all dependencies * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot --- packages/pandas-gbq/samples/snippets/requirements-test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements-test.txt b/packages/pandas-gbq/samples/snippets/requirements-test.txt index aedea97ec04c..96cc3163a065 100644 --- a/packages/pandas-gbq/samples/snippets/requirements-test.txt +++ b/packages/pandas-gbq/samples/snippets/requirements-test.txt @@ -1,2 +1,2 @@ google-cloud-testutils==1.4.0 -pytest==8.3.2 +pytest==8.3.3 From d90088114c63d35a1761f2798ac97f6bfe714180 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 23 Sep 2024 11:11:17 -0500 Subject: [PATCH 437/519] chore(main): release 0.23.2 (#806) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- packages/pandas-gbq/CHANGELOG.md | 14 ++++++++++++++ packages/pandas-gbq/pandas_gbq/version.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index ced3da531fb8..f5b132850134 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## [0.23.2](https://github.com/googleapis/python-bigquery-pandas/compare/v0.23.1...v0.23.2) (2024-09-20) + + +### Bug Fixes + +* **deps:** Require google-cloud-bigquery >= 3.4.2 ([5e14496](https://github.com/googleapis/python-bigquery-pandas/commit/5e144968893c476ffc9866461d128298e6b49d62)) +* **deps:** Require numpy >=1.18.1 ([5e14496](https://github.com/googleapis/python-bigquery-pandas/commit/5e144968893c476ffc9866461d128298e6b49d62)) +* **deps:** Require packaging >= 22.0 ([5e14496](https://github.com/googleapis/python-bigquery-pandas/commit/5e144968893c476ffc9866461d128298e6b49d62)) + + +### Documentation + +* Fix typo in 'vebosity' ([#803](https://github.com/googleapis/python-bigquery-pandas/issues/803)) ([a7641c9](https://github.com/googleapis/python-bigquery-pandas/commit/a7641c9b13be7f8649f43d985dac29cc7e05be0b)) + ## [0.23.1](https://github.com/googleapis/python-bigquery-pandas/compare/v0.23.0...v0.23.1) (2024-06-07) diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py index 2de80e86289c..73bc39aeed48 100644 --- a/packages/pandas-gbq/pandas_gbq/version.py +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -2,4 +2,4 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -__version__ = "0.23.1" +__version__ = "0.23.2" From b6ddccac7d1230bbfab22ed11d1184aa54754e8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Swe=C3=B1a=20=28Swast=29?= Date: Mon, 23 Sep 2024 12:18:28 -0500 Subject: [PATCH 438/519] fix!: `to_gbq` loads `unit8` columns to BigQuery INT64 instead of STRING (#814) * fix!: `to_gbq` loads `unit8` columns to BigQuery INT64 instead of STRING fix!: `to_gbq` loads naive (no timezone) columns to BigQuery DATETIME instead of TIMESTAMP (#814) fix!: `to_gbq` loads object column containing bool values to BOOLEAN instead of STRING (#814) fix!: `to_gbq` loads object column containing dictionary values to STRUCT instead of STRING (#814) deps: min pyarrow is now 4.0.0 to support compliant nested types (#814) Release-As: 0.24.0 --- packages/pandas-gbq/noxfile.py | 1 + packages/pandas-gbq/owlbot.py | 2 +- .../pandas-gbq/pandas_gbq/core/__init__.py | 3 + packages/pandas-gbq/pandas_gbq/core/pandas.py | 70 +++++ packages/pandas-gbq/pandas_gbq/gbq.py | 12 +- packages/pandas-gbq/pandas_gbq/load.py | 10 +- .../{schema.py => schema/__init__.py} | 31 -- .../pandas-gbq/pandas_gbq/schema/bigquery.py | 44 +++ .../pandas_gbq/schema/pandas_to_bigquery.py | 218 ++++++++++++++ .../pandas_gbq/schema/pyarrow_to_bigquery.py | 67 +++++ packages/pandas-gbq/setup.py | 5 +- .../pandas-gbq/testing/constraints-3.8.txt | 6 +- .../pandas-gbq/tests/system/test_to_gbq.py | 274 +++++++++++++++++- .../pandas-gbq/tests/unit/schema/__init__.py | 3 + .../unit/schema/test_pandas_to_bigquery.py | 156 ++++++++++ .../unit/schema/test_pyarrow_to_bigquery.py | 25 ++ packages/pandas-gbq/tests/unit/test_load.py | 5 +- packages/pandas-gbq/tests/unit/test_schema.py | 141 +++++++-- 18 files changed, 997 insertions(+), 76 deletions(-) create mode 100644 packages/pandas-gbq/pandas_gbq/core/__init__.py create mode 100644 packages/pandas-gbq/pandas_gbq/core/pandas.py rename packages/pandas-gbq/pandas_gbq/{schema.py => schema/__init__.py} (85%) create mode 100644 packages/pandas-gbq/pandas_gbq/schema/bigquery.py create mode 100644 packages/pandas-gbq/pandas_gbq/schema/pandas_to_bigquery.py create mode 100644 packages/pandas-gbq/pandas_gbq/schema/pyarrow_to_bigquery.py create mode 100644 packages/pandas-gbq/tests/unit/schema/__init__.py create mode 100644 packages/pandas-gbq/tests/unit/schema/test_pandas_to_bigquery.py create mode 100644 packages/pandas-gbq/tests/unit/schema/test_pyarrow_to_bigquery.py diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index d316dac8dc19..02cd052d83b6 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -51,6 +51,7 @@ UNIT_TEST_EXTRAS = [ "bqstorage", "tqdm", + "geopandas", ] UNIT_TEST_EXTRAS_BY_PYTHON = { "3.9": [], diff --git a/packages/pandas-gbq/owlbot.py b/packages/pandas-gbq/owlbot.py index 916a7074f1bd..190298a64c00 100644 --- a/packages/pandas-gbq/owlbot.py +++ b/packages/pandas-gbq/owlbot.py @@ -32,7 +32,7 @@ # Use a middle version of Python to test when no extras are installed. "3.9": [] } -extras = ["tqdm"] +extras = ["tqdm", "geopandas"] templated_files = common.py_library( unit_test_python_versions=["3.8", "3.9", "3.10", "3.11", "3.12"], system_test_python_versions=["3.8", "3.9", "3.10", "3.11", "3.12"], diff --git a/packages/pandas-gbq/pandas_gbq/core/__init__.py b/packages/pandas-gbq/pandas_gbq/core/__init__.py new file mode 100644 index 000000000000..02d26e8e8fb8 --- /dev/null +++ b/packages/pandas-gbq/pandas_gbq/core/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) 2024 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. diff --git a/packages/pandas-gbq/pandas_gbq/core/pandas.py b/packages/pandas-gbq/pandas_gbq/core/pandas.py new file mode 100644 index 000000000000..37557adf8b70 --- /dev/null +++ b/packages/pandas-gbq/pandas_gbq/core/pandas.py @@ -0,0 +1,70 @@ +# Copyright (c) 2019 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import itertools + +import pandas + + +def list_columns_and_indexes(dataframe, index=True): + """Return all index and column names with dtypes. + + Returns: + Sequence[Tuple[str, dtype]]: + Returns a sorted list of indexes and column names with + corresponding dtypes. If an index is missing a name or has the + same name as a column, the index is omitted. + """ + column_names = frozenset(dataframe.columns) + columns_and_indexes = [] + if index: + if isinstance(dataframe.index, pandas.MultiIndex): + for name in dataframe.index.names: + if name and name not in column_names: + values = dataframe.index.get_level_values(name) + columns_and_indexes.append((name, values.dtype)) + else: + if dataframe.index.name and dataframe.index.name not in column_names: + columns_and_indexes.append( + (dataframe.index.name, dataframe.index.dtype) + ) + + columns_and_indexes += zip(dataframe.columns, dataframe.dtypes) + return columns_and_indexes + + +def first_valid(series): + first_valid_index = series.first_valid_index() + if first_valid_index is not None: + return series.at[first_valid_index] + + +def first_array_valid(series): + """Return the first "meaningful" element from the array series. + + Here, "meaningful" means the first non-None element in one of the arrays that can + be used for type detextion. + """ + first_valid_index = series.first_valid_index() + if first_valid_index is None: + return None + + valid_array = series.at[first_valid_index] + valid_item = next((item for item in valid_array if not pandas.isna(item)), None) + + if valid_item is not None: + return valid_item + + # Valid item is None because all items in the "valid" array are invalid. Try + # to find a true valid array manually. + for array in itertools.islice(series, first_valid_index + 1, None): + try: + array_iter = iter(array) + except TypeError: + continue # Not an array, apparently, e.g. None, thus skip. + valid_item = next((item for item in array_iter if not pandas.isna(item)), None) + if valid_item is not None: + break + + return valid_item diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 19c42a6b04a3..06b6bbf228f6 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -25,6 +25,7 @@ from pandas_gbq.features import FEATURES import pandas_gbq.query import pandas_gbq.schema +import pandas_gbq.schema.pandas_to_bigquery import pandas_gbq.timestamp try: @@ -1219,9 +1220,16 @@ def _generate_bq_schema(df, default_type="STRING"): be overridden: https://github.com/pydata/pandas-gbq/issues/218, this method can be removed after there is time to migrate away from this method.""" - from pandas_gbq import schema + fields = pandas_gbq.schema.pandas_to_bigquery.dataframe_to_bigquery_fields( + df, + default_type=default_type, + ) + fields_json = [] + + for field in fields: + fields_json.append(field.to_api_repr()) - return schema.generate_bq_schema(df, default_type=default_type) + return {"fields": fields_json} class _Table(GbqConnector): diff --git a/packages/pandas-gbq/pandas_gbq/load.py b/packages/pandas-gbq/pandas_gbq/load.py index 45e474b2bd54..567899df664a 100644 --- a/packages/pandas-gbq/pandas_gbq/load.py +++ b/packages/pandas-gbq/pandas_gbq/load.py @@ -15,6 +15,8 @@ from pandas_gbq import exceptions import pandas_gbq.schema +import pandas_gbq.schema.bigquery +import pandas_gbq.schema.pandas_to_bigquery def encode_chunk(dataframe): @@ -214,11 +216,9 @@ def load_csv_from_file( This method is needed for writing with google-cloud-bigquery versions that don't implment load_table_from_dataframe with the CSV serialization format. """ - if schema is None: - schema = pandas_gbq.schema.generate_bq_schema(dataframe) - - schema = pandas_gbq.schema.remove_policy_tags(schema) - bq_schema = pandas_gbq.schema.to_google_cloud_bigquery(schema) + bq_schema = pandas_gbq.schema.pandas_to_bigquery.dataframe_to_bigquery_fields( + dataframe, schema + ) def load_chunk(chunk, job_config): try: diff --git a/packages/pandas-gbq/pandas_gbq/schema.py b/packages/pandas-gbq/pandas_gbq/schema/__init__.py similarity index 85% rename from packages/pandas-gbq/pandas_gbq/schema.py rename to packages/pandas-gbq/pandas_gbq/schema/__init__.py index b60fdedab795..350a1d2e394b 100644 --- a/packages/pandas-gbq/pandas_gbq/schema.py +++ b/packages/pandas-gbq/pandas_gbq/schema/__init__.py @@ -92,37 +92,6 @@ def schema_is_subset(schema_remote, schema_local): return all(field in fields_remote for field in fields_local) -def generate_bq_schema(dataframe, default_type="STRING"): - """Given a passed dataframe, generate the associated Google BigQuery schema. - - Arguments: - dataframe (pandas.DataFrame): D - default_type : string - The default big query type in case the type of the column - does not exist in the schema. - """ - - # If you update this mapping, also update the table at - # `docs/source/writing.rst`. - type_mapping = { - "i": "INTEGER", - "b": "BOOLEAN", - "f": "FLOAT", - "O": "STRING", - "S": "STRING", - "U": "STRING", - "M": "TIMESTAMP", - } - - fields = [] - for column_name, dtype in dataframe.dtypes.items(): - fields.append( - {"name": column_name, "type": type_mapping.get(dtype.kind, default_type)} - ) - - return {"fields": fields} - - def update_schema(schema_old, schema_new): """ Given an old BigQuery schema, update it with a new one. diff --git a/packages/pandas-gbq/pandas_gbq/schema/bigquery.py b/packages/pandas-gbq/pandas_gbq/schema/bigquery.py new file mode 100644 index 000000000000..0de21978bc4e --- /dev/null +++ b/packages/pandas-gbq/pandas_gbq/schema/bigquery.py @@ -0,0 +1,44 @@ +# Copyright (c) 2019 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import collections + +import google.cloud.bigquery + + +def to_schema_fields(schema): + """Coerce `schema` to a list of schema field instances. + + Args: + schema(Sequence[Union[ \ + :class:`~google.cloud.bigquery.schema.SchemaField`, \ + Mapping[str, Any] \ + ]]): + Table schema to convert. If some items are passed as mappings, + their content must be compatible with + :meth:`~google.cloud.bigquery.schema.SchemaField.from_api_repr`. + + Returns: + Sequence[:class:`~google.cloud.bigquery.schema.SchemaField`] + + Raises: + Exception: If ``schema`` is not a sequence, or if any item in the + sequence is not a :class:`~google.cloud.bigquery.schema.SchemaField` + instance or a compatible mapping representation of the field. + """ + for field in schema: + if not isinstance( + field, (google.cloud.bigquery.SchemaField, collections.abc.Mapping) + ): + raise ValueError( + "Schema items must either be fields or compatible " + "mapping representations." + ) + + return [ + field + if isinstance(field, google.cloud.bigquery.SchemaField) + else google.cloud.bigquery.SchemaField.from_api_repr(field) + for field in schema + ] diff --git a/packages/pandas-gbq/pandas_gbq/schema/pandas_to_bigquery.py b/packages/pandas-gbq/pandas_gbq/schema/pandas_to_bigquery.py new file mode 100644 index 000000000000..5a979a128e7b --- /dev/null +++ b/packages/pandas-gbq/pandas_gbq/schema/pandas_to_bigquery.py @@ -0,0 +1,218 @@ +# Copyright (c) 2019 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import collections.abc +import datetime +from typing import Optional, Tuple +import warnings + +import db_dtypes +from google.cloud.bigquery import schema +import pandas +import pyarrow + +import pandas_gbq.core.pandas +import pandas_gbq.schema.bigquery +import pandas_gbq.schema.pyarrow_to_bigquery + +try: + # _BaseGeometry is used to detect shapely objects in `bq_to_arrow_array` + from shapely.geometry.base import BaseGeometry as _BaseGeometry # type: ignore +except ImportError: + # No shapely, use NoneType for _BaseGeometry as a placeholder. + _BaseGeometry = type(None) + + +# If you update this mapping, also update the table at +# `docs/source/writing.rst`. +_PANDAS_DTYPE_TO_BQ = { + "bool": "BOOLEAN", + "datetime64[ns, UTC]": "TIMESTAMP", + "datetime64[ns]": "DATETIME", + "float32": "FLOAT", + "float64": "FLOAT", + "int8": "INTEGER", + "int16": "INTEGER", + "int32": "INTEGER", + "int64": "INTEGER", + "uint8": "INTEGER", + "uint16": "INTEGER", + "uint32": "INTEGER", + "geometry": "GEOGRAPHY", + db_dtypes.DateDtype.name: "DATE", + db_dtypes.TimeDtype.name: "TIME", + # TODO(tswast): Add support for JSON. +} + + +def dataframe_to_bigquery_fields( + dataframe, + override_bigquery_fields=None, + default_type="STRING", + index=False, +) -> Tuple[schema.SchemaField]: + """Convert a pandas DataFrame schema to a BigQuery schema. + + Args: + dataframe (pandas.DataFrame): + DataFrame for which the client determines the BigQuery schema. + override_bigquery_fields (Sequence[Union[ \ + :class:`~google.cloud.bigquery.schema.SchemaField`, \ + Mapping[str, Any] \ + ]]): + A BigQuery schema. Use this argument to override the autodetected + type for some or all of the DataFrame columns. + + Returns: + Optional[Sequence[google.cloud.bigquery.schema.SchemaField]]: + The automatically determined schema. Returns None if the type of + any column cannot be determined. + """ + if override_bigquery_fields: + override_bigquery_fields = pandas_gbq.schema.bigquery.to_schema_fields( + override_bigquery_fields + ) + override_fields_by_name = { + field.name: field for field in override_bigquery_fields + } + override_fields_unused = set(override_fields_by_name.keys()) + else: + override_fields_by_name = {} + override_fields_unused = set() + + bq_schema_out = [] + unknown_type_fields = [] + + # TODO(tswast): Support index=True in to_gbq. + for column, dtype in pandas_gbq.core.pandas.list_columns_and_indexes( + dataframe, index=index + ): + # Use provided type from schema, if present. + bq_field = override_fields_by_name.get(column) + if bq_field: + bq_schema_out.append(bq_field) + override_fields_unused.discard(bq_field.name) + continue + + # Try to automatically determine the type based on the pandas dtype. + bq_field = dtype_to_bigquery_field(column, dtype) + if bq_field: + bq_schema_out.append(bq_field) + continue + + # Try to automatically determine the type based on a few rows of the data. + values = dataframe.reset_index()[column] + bq_field = values_to_bigquery_field(column, values) + + if bq_field: + bq_schema_out.append(bq_field) + continue + + # Try to automatically determine the type based on the arrow conversion. + try: + arrow_value = pyarrow.array(values) + bq_field = ( + pandas_gbq.schema.pyarrow_to_bigquery.arrow_type_to_bigquery_field( + column, arrow_value.type + ) + ) + + if bq_field: + bq_schema_out.append(bq_field) + continue + except pyarrow.lib.ArrowInvalid: + # TODO(tswast): Better error message if conversion to arrow fails. + pass + + # Unknown field type. + bq_field = schema.SchemaField(column, default_type) + bq_schema_out.append(bq_field) + unknown_type_fields.append(bq_field) + + # Catch any schema mismatch. The developer explicitly asked to serialize a + # column, but it was not found. + if override_fields_unused: + raise ValueError( + "Provided BigQuery fields contain field(s) not present in DataFrame: {}".format( + override_fields_unused + ) + ) + + # If schema detection was not successful for all columns, also try with + # pyarrow, if available. + if unknown_type_fields: + msg = "Could not determine the type of columns: {}".format( + ", ".join(field.name for field in unknown_type_fields) + ) + warnings.warn(msg) + + return tuple(bq_schema_out) + + +def dtype_to_bigquery_field(name, dtype) -> Optional[schema.SchemaField]: + bq_type = _PANDAS_DTYPE_TO_BQ.get(dtype.name) + + if bq_type is not None: + return schema.SchemaField(name, bq_type) + + if hasattr(pandas, "ArrowDtype") and isinstance(dtype, pandas.ArrowDtype): + return pandas_gbq.schema.pyarrow_to_bigquery.arrow_type_to_bigquery_field( + name, dtype.pyarrow_dtype + ) + + return None + + +def value_to_bigquery_field(name, value) -> Optional[schema.SchemaField]: + if isinstance(value, str): + return schema.SchemaField(name, "STRING") + + # For timezone-naive datetimes, the later pyarrow conversion to try and + # learn the type add a timezone to such datetimes, causing them to be + # recognized as TIMESTAMP type. We thus additionally check the actual data + # to see if we need to overrule that and choose DATETIME instead. + # + # See: https://github.com/googleapis/python-bigquery/issues/985 + # and https://github.com/googleapis/python-bigquery/pull/1061 + # and https://github.com/googleapis/python-bigquery-pandas/issues/450 + if isinstance(value, datetime.datetime): + if value.tzinfo is not None: + return schema.SchemaField(name, "TIMESTAMP") + else: + return schema.SchemaField(name, "DATETIME") + + if _BaseGeometry is not None and isinstance(value, _BaseGeometry): + return schema.SchemaField(name, "GEOGRAPHY") + + return None + + +def values_to_bigquery_field(name, values) -> Optional[schema.SchemaField]: + value = pandas_gbq.core.pandas.first_valid(values) + + # All NULL, type not determinable. + if value is None: + return None + + field = value_to_bigquery_field(name, value) + if field is not None: + return field + + if isinstance(value, str): + return schema.SchemaField(name, "STRING") + + # Check plain ARRAY values here. Let STRUCT get determined by pyarrow, + # which can examine more values to determine all keys. + if isinstance(value, collections.abc.Iterable) and not isinstance( + value, collections.abc.Mapping + ): + # It could be that this value contains all None or is empty, so get the + # first non-None value we can find. + valid_item = pandas_gbq.core.pandas.first_array_valid(values) + field = value_to_bigquery_field(name, valid_item) + + if field is not None: + return schema.SchemaField(name, field.field_type, mode="REPEATED") + + return None diff --git a/packages/pandas-gbq/pandas_gbq/schema/pyarrow_to_bigquery.py b/packages/pandas-gbq/pandas_gbq/schema/pyarrow_to_bigquery.py new file mode 100644 index 000000000000..c63559ebff72 --- /dev/null +++ b/packages/pandas-gbq/pandas_gbq/schema/pyarrow_to_bigquery.py @@ -0,0 +1,67 @@ +# Copyright (c) 2023 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +from typing import Optional, cast + +from google.cloud.bigquery import schema +import pyarrow +import pyarrow.types + +_ARROW_SCALAR_IDS_TO_BQ = { + # https://arrow.apache.org/docs/python/api/datatypes.html#type-classes + pyarrow.bool_().id: "BOOLEAN", + pyarrow.int8().id: "INTEGER", + pyarrow.int16().id: "INTEGER", + pyarrow.int32().id: "INTEGER", + pyarrow.int64().id: "INTEGER", + pyarrow.uint8().id: "INTEGER", + pyarrow.uint16().id: "INTEGER", + pyarrow.uint32().id: "INTEGER", + pyarrow.uint64().id: "INTEGER", + pyarrow.float16().id: "FLOAT", + pyarrow.float32().id: "FLOAT", + pyarrow.float64().id: "FLOAT", + pyarrow.time32("ms").id: "TIME", + pyarrow.time64("ns").id: "TIME", + pyarrow.timestamp("ns").id: "TIMESTAMP", + pyarrow.date32().id: "DATE", + pyarrow.date64().id: "DATETIME", # because millisecond resolution + pyarrow.binary().id: "BYTES", + pyarrow.string().id: "STRING", # also alias for pyarrow.utf8() + pyarrow.large_string().id: "STRING", + # The exact decimal's scale and precision are not important, as only + # the type ID matters, and it's the same for all decimal256 instances. + pyarrow.decimal128(38, scale=9).id: "NUMERIC", + pyarrow.decimal256(76, scale=38).id: "BIGNUMERIC", +} + + +def arrow_type_to_bigquery_field(name, type_) -> Optional[schema.SchemaField]: + detected_type = _ARROW_SCALAR_IDS_TO_BQ.get(type_.id, None) + if detected_type is not None: + return schema.SchemaField(name, detected_type) + + if pyarrow.types.is_list(type_): + return arrow_list_type_to_bigquery(name, type_) + + if pyarrow.types.is_struct(type_): + inner_fields: list[pyarrow.Field] = [] + struct_type = cast(pyarrow.StructType, type_) + for field_index in range(struct_type.num_fields): + field = struct_type[field_index] + inner_fields.append(arrow_type_to_bigquery_field(field.name, field.type)) + + return schema.SchemaField(name, "RECORD", fields=inner_fields) + + return None + + +def arrow_list_type_to_bigquery(name, type_) -> Optional[schema.SchemaField]: + inner_field = arrow_type_to_bigquery_field(name, type_.value_type) + if inner_field is None: + return None + + return schema.SchemaField( + name, inner_field.field_type, mode="REPEATED", fields=inner_field.fields + ) diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index df793e59779c..10d977333d59 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -25,7 +25,7 @@ "db-dtypes >=1.0.4,<2.0.0", "numpy >=1.18.1", "pandas >=1.1.4", - "pyarrow >=3.0.0", + "pyarrow >=4.0.0", "pydata-google-auth >=1.5.0", # Note: google-api-core and google-auth are also included via transitive # dependency on google-cloud-bigquery, but this library also uses them @@ -42,7 +42,8 @@ "bqstorage": [ "google-cloud-bigquery-storage >=2.16.2, <3.0.0dev", ], - "tqdm": "tqdm>=4.23.0", + "tqdm": ["tqdm>=4.23.0"], + "geopandas": ["geopandas>=0.9.0", "Shapely>=1.8.4"], } # Setup boilerplate below this line. diff --git a/packages/pandas-gbq/testing/constraints-3.8.txt b/packages/pandas-gbq/testing/constraints-3.8.txt index e551d17e3e92..8d6ef4f493db 100644 --- a/packages/pandas-gbq/testing/constraints-3.8.txt +++ b/packages/pandas-gbq/testing/constraints-3.8.txt @@ -7,6 +7,7 @@ # Then this file should have foo==1.14.0 # protobuf==3.19.5 db-dtypes==1.0.4 +geopandas==0.9.0 google-api-core==2.10.2 google-auth==2.13.0 google-auth-oauthlib==0.7.0 @@ -14,7 +15,8 @@ google-cloud-bigquery==3.4.2 google-cloud-bigquery-storage==2.16.2 numpy==1.18.1 pandas==1.1.4 -pyarrow==3.0.0 +pyarrow==4.0.0 pydata-google-auth==1.5.0 +Shapely==1.8.4 tqdm==4.23.0 -packaging==22.0.0 \ No newline at end of file +packaging==22.0.0 diff --git a/packages/pandas-gbq/tests/system/test_to_gbq.py b/packages/pandas-gbq/tests/system/test_to_gbq.py index 2e7245d5575b..6352fbd7620d 100644 --- a/packages/pandas-gbq/tests/system/test_to_gbq.py +++ b/packages/pandas-gbq/tests/system/test_to_gbq.py @@ -10,6 +10,7 @@ import db_dtypes import pandas import pandas.testing +import pyarrow import pytest pytest.importorskip("google.cloud.bigquery", minversion="1.24.0") @@ -125,6 +126,37 @@ def test_series_round_trip( ) DATAFRAME_ROUND_TRIPS = [ + # Ensure that a BOOLEAN column can be written with bool, boolean, and + # object dtypes. See: + # https://github.com/googleapis/python-bigquery-pandas/issues/105 + pytest.param( + *DataFrameRoundTripTestCase( + input_df=pandas.DataFrame( + { + "row_num": [0, 1, 2], + "bool_col": pandas.Series( + [True, False, True], + dtype="bool", + ), + "boolean_col": pandas.Series( + [None, True, False], + dtype="boolean", + ), + "object_col": pandas.Series( + [False, None, True], + dtype="object", + ), + } + ), + table_schema=[ + {"name": "bool_col", "type": "BOOLEAN"}, + {"name": "boolean_col", "type": "BOOLEAN"}, + {"name": "object_col", "type": "BOOLEAN"}, + ], + api_methods={"load_csv", "load_parquet"}, + ), + id="boolean", + ), # Ensure that a DATE column can be written with datetime64[ns] dtype # data. See: # https://github.com/googleapis/python-bigquery-pandas/issues/362 @@ -176,6 +208,96 @@ def test_series_round_trip( {"name": "date_col", "type": "DATE"}, ], ), + # Loading an INTEGER column should work for any integer dtype. See: + # https://github.com/googleapis/python-bigquery-pandas/issues/616 + pytest.param( + *DataFrameRoundTripTestCase( + input_df=pandas.DataFrame( + { + "row_num": [0, 1, 2], + "object": pandas.Series( + [None, 1, -2], + dtype="object", + ), + "nullable_int64": pandas.Series( + [3, None, -4], + dtype="Int64", + ), + "int8": pandas.Series( + [5, -6, 7], + dtype="int8", + ), + "int16": pandas.Series( + [-8, 9, -10], + dtype="int16", + ), + "int32": pandas.Series( + [11, -12, 13], + dtype="int32", + ), + "int64": pandas.Series( + [-14, 15, -16], + dtype="int64", + ), + "uint8": pandas.Series( + [0, 1, 2], + dtype="uint8", + ), + "uint16": pandas.Series( + [3, 4, 5], + dtype="uint16", + ), + "uint32": pandas.Series( + [6, 7, 8], + dtype="uint32", + ), + } + ), + expected_df=pandas.DataFrame( + { + "row_num": [0, 1, 2], + "object": pandas.Series( + [None, 1, -2], + dtype="Int64", + ), + "nullable_int64": pandas.Series( + [3, None, -4], + dtype="Int64", + ), + "int8": pandas.Series( + [5, -6, 7], + dtype="Int64", + ), + "int16": pandas.Series( + [-8, 9, -10], + dtype="Int64", + ), + "int32": pandas.Series( + [11, -12, 13], + dtype="Int64", + ), + "int64": pandas.Series( + [-14, 15, -16], + dtype="Int64", + ), + "uint8": pandas.Series( + [0, 1, 2], + dtype="Int64", + ), + "uint16": pandas.Series( + [3, 4, 5], + dtype="Int64", + ), + "uint32": pandas.Series( + [6, 7, 8], + dtype="Int64", + ), + } + ), + api_methods={"load_csv", "load_parquet"}, + ), + id="integer", + ), # Loading a NUMERIC column should work for floating point objects. See: # https://github.com/googleapis/python-bigquery-pandas/issues/421 DataFrameRoundTripTestCase( @@ -240,6 +362,133 @@ def test_series_round_trip( ), id="issue365-extreme-datetimes", ), + pytest.param( + # Load STRUCT and ARRAY using either object column or ArrowDtype. + # See: https://github.com/googleapis/python-bigquery-pandas/issues/452 + *DataFrameRoundTripTestCase( + input_df=pandas.DataFrame( + { + "row_num": [0, 1, 2], + "object_struct": pandas.Series( + [{"test": "str1"}, {"test": "str2"}, {"test": "str3"}], + dtype="object", + ), + # Array of DATETIME requires inspection into list elements. + # See: + # https://github.com/googleapis/python-bigquery/pull/1061 + "object_array_datetime": pandas.Series( + [[], [datetime.datetime(1998, 9, 4, 12, 0, 0)], []], + dtype="object", + ), + "object_array_of_struct": pandas.Series( + [[], [{"test": "str4"}], []], dtype="object" + ), + "arrow_struct": pandas.Series( + [ + {"version": 1, "project": "pandas"}, + {"version": 2, "project": "pandas"}, + {"version": 1, "project": "numpy"}, + ], + dtype=pandas.ArrowDtype( + pyarrow.struct( + [ + ("version", pyarrow.int64()), + ("project", pyarrow.string()), + ] + ) + ) + if hasattr(pandas, "ArrowDtype") + else "object", + ), + "arrow_array": pandas.Series( + [[1, 2, 3], None, [4, 5, 6]], + dtype=pandas.ArrowDtype( + pyarrow.list_(pyarrow.int64()), + ) + if hasattr(pandas, "ArrowDtype") + else "object", + ), + "arrow_array_of_struct": pandas.Series( + [ + [{"test": "str5"}], + None, + [{"test": "str6"}, {"test": "str7"}], + ], + dtype=pandas.ArrowDtype( + pyarrow.list_(pyarrow.struct([("test", pyarrow.string())])), + ) + if hasattr(pandas, "ArrowDtype") + else "object", + ), + }, + ), + expected_df=pandas.DataFrame( + { + "row_num": [0, 1, 2], + "object_struct": pandas.Series( + [{"test": "str1"}, {"test": "str2"}, {"test": "str3"}], + dtype=pandas.ArrowDtype( + pyarrow.struct([("test", pyarrow.string())]), + ) + if hasattr(pandas, "ArrowDtype") + else "object", + ), + # Array of DATETIME requires inspection into list elements. + # See: + # https://github.com/googleapis/python-bigquery/pull/1061 + "object_array_datetime": pandas.Series( + [[], [datetime.datetime(1998, 9, 4, 12, 0, 0)], []], + dtype=pandas.ArrowDtype(pyarrow.list_(pyarrow.timestamp("us"))) + if hasattr(pandas, "ArrowDtype") + else "object", + ), + "object_array_of_struct": pandas.Series( + [[], [{"test": "str4"}], []], + dtype=pandas.ArrowDtype( + pyarrow.list_(pyarrow.struct([("test", pyarrow.string())])), + ) + if hasattr(pandas, "ArrowDtype") + else "object", + ), + "arrow_struct": pandas.Series( + [ + {"version": 1, "project": "pandas"}, + {"version": 2, "project": "pandas"}, + {"version": 1, "project": "numpy"}, + ], + dtype=pandas.ArrowDtype( + pyarrow.struct( + [ + ("version", pyarrow.int64()), + ("project", pyarrow.string()), + ] + ) + ) + if hasattr(pandas, "ArrowDtype") + else "object", + ), + "arrow_array": pandas.Series( + [[1, 2, 3], [], [4, 5, 6]], + dtype=pandas.ArrowDtype( + pyarrow.list_(pyarrow.int64()), + ) + if hasattr(pandas, "ArrowDtype") + else "object", + ), + "arrow_array_of_struct": pandas.Series( + [[{"test": "str5"}], [], [{"test": "str6"}, {"test": "str7"}]], + dtype=pandas.ArrowDtype( + pyarrow.list_(pyarrow.struct([("test", pyarrow.string())])), + ) + if hasattr(pandas, "ArrowDtype") + else "object", + ), + }, + ), + api_methods={"load_parquet"}, + ), + id="struct", + ), ] @@ -264,13 +513,20 @@ def test_dataframe_round_trip_with_table_schema( method_under_test( input_df, table_id, table_schema=table_schema, api_method=api_method ) - round_trip = read_gbq( - table_id, - dtypes=dict(zip(expected_df.columns, expected_df.dtypes)), - # BigQuery Storage API is required to avoid out-of-bound due to extra - # day from rounding error which was fixed in google-cloud-bigquery - # 2.6.0. https://github.com/googleapis/python-bigquery/pull/402 - use_bqstorage_api=True, + round_trip = ( + read_gbq( + table_id, + dtypes=dict(zip(expected_df.columns, expected_df.dtypes)), + # BigQuery Storage API is required to avoid out-of-bound due to extra + # day from rounding error which was fixed in google-cloud-bigquery + # 2.6.0. https://github.com/googleapis/python-bigquery/pull/402 + use_bqstorage_api=True, + ) + .set_index("row_num") + .sort_index() + ) + + # TODO(tswast): Support writing index columns if to_gbq(index=True). + pandas.testing.assert_frame_equal( + expected_df.set_index("row_num").sort_index(), round_trip ) - round_trip.sort_values("row_num", inplace=True) - pandas.testing.assert_frame_equal(expected_df, round_trip) diff --git a/packages/pandas-gbq/tests/unit/schema/__init__.py b/packages/pandas-gbq/tests/unit/schema/__init__.py new file mode 100644 index 000000000000..02d26e8e8fb8 --- /dev/null +++ b/packages/pandas-gbq/tests/unit/schema/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) 2024 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. diff --git a/packages/pandas-gbq/tests/unit/schema/test_pandas_to_bigquery.py b/packages/pandas-gbq/tests/unit/schema/test_pandas_to_bigquery.py new file mode 100644 index 000000000000..924ce1eecab2 --- /dev/null +++ b/packages/pandas-gbq/tests/unit/schema/test_pandas_to_bigquery.py @@ -0,0 +1,156 @@ +# Copyright (c) 2019 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import collections +import datetime +import operator + +from google.cloud.bigquery import schema +import pandas +import pytest + + +@pytest.fixture +def module_under_test(): + from pandas_gbq.schema import pandas_to_bigquery + + return pandas_to_bigquery + + +def test_dataframe_to_bigquery_fields_w_named_index(module_under_test): + df_data = collections.OrderedDict( + [ + ("str_column", ["hello", "world"]), + ("int_column", [42, 8]), + ("bool_column", [True, False]), + ] + ) + index = pandas.Index(["a", "b"], name="str_index") + dataframe = pandas.DataFrame(df_data, index=index) + + returned_schema = module_under_test.dataframe_to_bigquery_fields( + dataframe, [], index=True + ) + + expected_schema = ( + schema.SchemaField("str_index", "STRING", "NULLABLE"), + schema.SchemaField("str_column", "STRING", "NULLABLE"), + schema.SchemaField("int_column", "INTEGER", "NULLABLE"), + schema.SchemaField("bool_column", "BOOLEAN", "NULLABLE"), + ) + assert returned_schema == expected_schema + + +def test_dataframe_to_bigquery_fields_w_multiindex(module_under_test): + df_data = collections.OrderedDict( + [ + ("str_column", ["hello", "world"]), + ("int_column", [42, 8]), + ("bool_column", [True, False]), + ] + ) + index = pandas.MultiIndex.from_tuples( + [ + ("a", 0, datetime.datetime(1999, 12, 31, 23, 59, 59, 999999)), + ("a", 0, datetime.datetime(2000, 1, 1, 0, 0, 0)), + ], + names=["str_index", "int_index", "dt_index"], + ) + dataframe = pandas.DataFrame(df_data, index=index) + + returned_schema = module_under_test.dataframe_to_bigquery_fields( + dataframe, [], index=True + ) + + expected_schema = ( + schema.SchemaField("str_index", "STRING", "NULLABLE"), + schema.SchemaField("int_index", "INTEGER", "NULLABLE"), + schema.SchemaField("dt_index", "DATETIME", "NULLABLE"), + schema.SchemaField("str_column", "STRING", "NULLABLE"), + schema.SchemaField("int_column", "INTEGER", "NULLABLE"), + schema.SchemaField("bool_column", "BOOLEAN", "NULLABLE"), + ) + assert returned_schema == expected_schema + + +def test_dataframe_to_bigquery_fields_w_bq_schema(module_under_test): + df_data = collections.OrderedDict( + [ + ("str_column", ["hello", "world"]), + ("int_column", [42, 8]), + ("bool_column", [True, False]), + ] + ) + dataframe = pandas.DataFrame(df_data) + + dict_schema = [ + {"name": "str_column", "type": "STRING", "mode": "NULLABLE"}, + {"name": "bool_column", "type": "BOOL", "mode": "REQUIRED"}, + ] + + returned_schema = module_under_test.dataframe_to_bigquery_fields( + dataframe, dict_schema + ) + + expected_schema = ( + schema.SchemaField("str_column", "STRING", "NULLABLE"), + schema.SchemaField("int_column", "INTEGER", "NULLABLE"), + schema.SchemaField("bool_column", "BOOL", "REQUIRED"), + ) + assert returned_schema == expected_schema + + +def test_dataframe_to_bigquery_fields_fallback_needed_w_pyarrow(module_under_test): + dataframe = pandas.DataFrame( + data=[ + {"id": 10, "status": "FOO", "created_at": datetime.date(2019, 5, 10)}, + {"id": 20, "status": "BAR", "created_at": datetime.date(2018, 9, 12)}, + ] + ) + + detected_schema = module_under_test.dataframe_to_bigquery_fields( + dataframe, override_bigquery_fields=[] + ) + expected_schema = ( + schema.SchemaField("id", "INTEGER", mode="NULLABLE"), + schema.SchemaField("status", "STRING", mode="NULLABLE"), + schema.SchemaField("created_at", "DATE", mode="NULLABLE"), + ) + by_name = operator.attrgetter("name") + assert sorted(detected_schema, key=by_name) == sorted(expected_schema, key=by_name) + + +def test_dataframe_to_bigquery_fields_w_extra_fields(module_under_test): + with pytest.raises(ValueError) as exc_context: + module_under_test.dataframe_to_bigquery_fields( + pandas.DataFrame(), + override_bigquery_fields=(schema.SchemaField("not_in_df", "STRING"),), + ) + message = str(exc_context.value) + assert ( + "Provided BigQuery fields contain field(s) not present in DataFrame:" in message + ) + assert "not_in_df" in message + + +def test_dataframe_to_bigquery_fields_geography(module_under_test): + geopandas = pytest.importorskip("geopandas") + from shapely import wkt + + df = geopandas.GeoDataFrame( + pandas.DataFrame( + dict( + name=["foo", "bar"], + geo1=[None, None], + geo2=[None, wkt.loads("Point(1 1)")], + ) + ), + geometry="geo1", + ) + bq_schema = module_under_test.dataframe_to_bigquery_fields(df, []) + assert bq_schema == ( + schema.SchemaField("name", "STRING"), + schema.SchemaField("geo1", "GEOGRAPHY"), + schema.SchemaField("geo2", "GEOGRAPHY"), + ) diff --git a/packages/pandas-gbq/tests/unit/schema/test_pyarrow_to_bigquery.py b/packages/pandas-gbq/tests/unit/schema/test_pyarrow_to_bigquery.py new file mode 100644 index 000000000000..9a20e3426ec5 --- /dev/null +++ b/packages/pandas-gbq/tests/unit/schema/test_pyarrow_to_bigquery.py @@ -0,0 +1,25 @@ +# Copyright (c) 2024 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import pyarrow + +from pandas_gbq.schema import pyarrow_to_bigquery + + +def test_arrow_type_to_bigquery_field_unknown(): + # Default types should be picked at a higher layer. + assert ( + pyarrow_to_bigquery.arrow_type_to_bigquery_field("test_name", pyarrow.null()) + is None + ) + + +def test_arrow_type_to_bigquery_field_list_of_unknown(): + # Default types should be picked at a higher layer. + assert ( + pyarrow_to_bigquery.arrow_type_to_bigquery_field( + "test_name", pyarrow.list_(pyarrow.null()) + ) + is None + ) diff --git a/packages/pandas-gbq/tests/unit/test_load.py b/packages/pandas-gbq/tests/unit/test_load.py index 45c735336180..bb6117817b1d 100644 --- a/packages/pandas-gbq/tests/unit/test_load.py +++ b/packages/pandas-gbq/tests/unit/test_load.py @@ -165,11 +165,8 @@ def test_load_csv_from_file_generates_schema(mock_bigquery_client): assert sent_schema[2].field_type == "FLOAT" assert sent_schema[3].name == "string_col" assert sent_schema[3].field_type == "STRING" - # TODO: Disambiguate TIMESTAMP from DATETIME based on if column is - # localized or at least use field type from table metadata. See: - # https://github.com/googleapis/python-bigquery-pandas/issues/450 assert sent_schema[4].name == "datetime_col" - assert sent_schema[4].field_type == "TIMESTAMP" + assert sent_schema[4].field_type == "DATETIME" assert sent_schema[5].name == "timestamp_col" assert sent_schema[5].field_type == "TIMESTAMP" diff --git a/packages/pandas-gbq/tests/unit/test_schema.py b/packages/pandas-gbq/tests/unit/test_schema.py index 7fdc616cdec9..48e8862a8174 100644 --- a/packages/pandas-gbq/tests/unit/test_schema.py +++ b/packages/pandas-gbq/tests/unit/test_schema.py @@ -7,14 +7,12 @@ import google.cloud.bigquery import pandas +import pyarrow import pytest - -@pytest.fixture -def module_under_test(): - import pandas_gbq.schema - - return pandas_gbq.schema +import pandas_gbq +import pandas_gbq.gbq +import pandas_gbq.schema @pytest.mark.parametrize( @@ -45,17 +43,15 @@ def module_under_test(): ), ], ) -def test_schema_is_subset_passes_if_subset( - module_under_test, original_fields, dataframe_fields -): +def test_schema_is_subset_passes_if_subset(original_fields, dataframe_fields): # Issue #24 schema_is_subset indicates whether the schema of the # dataframe is a subset of the schema of the bigquery table table_schema = {"fields": original_fields} tested_schema = {"fields": dataframe_fields} - assert module_under_test.schema_is_subset(table_schema, tested_schema) + assert pandas_gbq.schema.schema_is_subset(table_schema, tested_schema) -def test_schema_is_subset_fails_if_not_subset(module_under_test): +def test_schema_is_subset_fails_if_not_subset(): table_schema = { "fields": [ {"name": "A", "type": "FLOAT"}, @@ -66,12 +62,17 @@ def test_schema_is_subset_fails_if_not_subset(module_under_test): tested_schema = { "fields": [{"name": "A", "type": "FLOAT"}, {"name": "C", "type": "FLOAT"}] } - assert not module_under_test.schema_is_subset(table_schema, tested_schema) + assert not pandas_gbq.schema.schema_is_subset(table_schema, tested_schema) @pytest.mark.parametrize( "dataframe,expected_schema", [ + pytest.param( + pandas.DataFrame(data={"col1": [object()]}), + {"fields": [{"name": "col1", "type": "STRING"}]}, + id="default-type-fails-pyarrow-conversion", + ), ( pandas.DataFrame(data={"col1": [1, 2, 3]}), {"fields": [{"name": "col1", "type": "INTEGER"}]}, @@ -88,13 +89,39 @@ def test_schema_is_subset_fails_if_not_subset(module_under_test): pandas.DataFrame(data={"col1": ["hello", "world"]}), {"fields": [{"name": "col1", "type": "STRING"}]}, ), - ( - pandas.DataFrame(data={"col1": [datetime.datetime.now()]}), - {"fields": [{"name": "col1", "type": "TIMESTAMP"}]}, + pytest.param( + # No time zone -> DATETIME, + # Time zone -> TIMESTAMP + # See: https://github.com/googleapis/python-bigquery-pandas/issues/450 + pandas.DataFrame( + data={ + "object1": pandas.Series([datetime.datetime.now()], dtype="object"), + "object2": pandas.Series( + [datetime.datetime.now(datetime.timezone.utc)], dtype="object" + ), + "datetime1": pandas.Series( + [datetime.datetime.now()], dtype="datetime64[ns]" + ), + "datetime2": pandas.Series( + [datetime.datetime.now(datetime.timezone.utc)], + dtype="datetime64[ns, UTC]", + ), + } + ), + { + "fields": [ + {"name": "object1", "type": "DATETIME"}, + {"name": "object2", "type": "TIMESTAMP"}, + {"name": "datetime1", "type": "DATETIME"}, + {"name": "datetime2", "type": "TIMESTAMP"}, + ] + }, + id="issue450-datetime", ), ( pandas.DataFrame( data={ + "col0": [datetime.datetime.now(datetime.timezone.utc)], "col1": [datetime.datetime.now()], "col2": ["hello"], "col3": [3.14], @@ -104,7 +131,8 @@ def test_schema_is_subset_fails_if_not_subset(module_under_test): ), { "fields": [ - {"name": "col1", "type": "TIMESTAMP"}, + {"name": "col0", "type": "TIMESTAMP"}, + {"name": "col1", "type": "DATETIME"}, {"name": "col2", "type": "STRING"}, {"name": "col3", "type": "FLOAT"}, {"name": "col4", "type": "BOOLEAN"}, @@ -112,10 +140,83 @@ def test_schema_is_subset_fails_if_not_subset(module_under_test): ] }, ), + pytest.param( + # uint8, which is the result from get_dummies, should be INTEGER. + # https://github.com/googleapis/python-bigquery-pandas/issues/616 + pandas.DataFrame({"col": [0, 1]}, dtype="uint8"), + {"fields": [{"name": "col", "type": "INTEGER"}]}, + id="issue616-uint8", + ), + pytest.param( + # object column containing dictionaries should load to STRUCT. + # https://github.com/googleapis/python-bigquery-pandas/issues/452 + pandas.DataFrame( + { + "my_struct": pandas.Series( + [{"test": "str1"}, {"test": "str2"}, {"test": "str3"}], + dtype="object", + ), + } + ), + { + "fields": [ + { + "name": "my_struct", + "type": "RECORD", + "fields": [ + {"name": "test", "type": "STRING", "mode": "NULLABLE"} + ], + } + ] + }, + id="issue452-struct", + ), + pytest.param( + pandas.DataFrame( + { + "object": pandas.Series([[], ["abc"], []], dtype="object"), + "list": pandas.Series( + [[], [1, 2, 3], []], + dtype=pandas.ArrowDtype(pyarrow.list_(pyarrow.int64())) + if hasattr(pandas, "ArrowDtype") + else "object", + ), + "list_of_struct": pandas.Series( + [[], [{"test": "abc"}], []], + dtype=pandas.ArrowDtype( + pyarrow.list_(pyarrow.struct([("test", pyarrow.string())])) + ) + if hasattr(pandas, "ArrowDtype") + else "object", + ), + } + ), + { + "fields": [ + {"name": "object", "type": "STRING", "mode": "REPEATED"}, + {"name": "list", "type": "INTEGER", "mode": "REPEATED"}, + { + "name": "list_of_struct", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + {"name": "test", "type": "STRING", "mode": "NULLABLE"}, + ], + }, + ], + }, + id="array", + ), ], ) -def test_generate_bq_schema(module_under_test, dataframe, expected_schema): - schema = module_under_test.generate_bq_schema(dataframe) +def test_generate_bq_schema(dataframe, expected_schema): + schema = pandas_gbq.gbq._generate_bq_schema(dataframe) + + # NULLABLE is the default mode. + for field in expected_schema["fields"]: + if "mode" not in field: + field["mode"] = "NULLABLE" + assert schema == expected_schema @@ -156,8 +257,8 @@ def test_generate_bq_schema(module_under_test, dataframe, expected_schema): ), ], ) -def test_update_schema(module_under_test, schema_old, schema_new, expected_output): - output = module_under_test.update_schema(schema_old, schema_new) +def test_update_schema(schema_old, schema_new, expected_output): + output = pandas_gbq.schema.update_schema(schema_old, schema_new) assert output == expected_output From 1e3e204d2d366de76a5a94fde5c3b14b8684ff45 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Mon, 7 Oct 2024 06:51:53 -0400 Subject: [PATCH 439/519] chore: adds Python 3.7/3.8 EOL pending deprecation warning (#817) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: adds deprecation warnings related to Python 3.7 3.8 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * Update pandas_gbq/__init__.py --------- Co-authored-by: Owl Bot --- packages/pandas-gbq/noxfile.py | 5 +++ packages/pandas-gbq/pandas_gbq/__init__.py | 14 ++++++++ .../pandas_gbq/_versions_helpers.py | 32 +++++++++++++++++++ 3 files changed, 51 insertions(+) create mode 100644 packages/pandas-gbq/pandas_gbq/_versions_helpers.py diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 02cd052d83b6..461b761c6a7d 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -208,6 +208,7 @@ def default(session): session.run( "py.test", "--quiet", + "-W default::PendingDeprecationWarning", f"--junitxml=unit_{session.python}_sponge_log.xml", "--cov=pandas_gbq", "--cov=tests/unit", @@ -290,6 +291,7 @@ def system(session): session.run( "py.test", "--quiet", + "-W default::PendingDeprecationWarning", f"--junitxml=system_{session.python}_sponge_log.xml", system_test_path, *session.posargs, @@ -298,6 +300,7 @@ def system(session): session.run( "py.test", "--quiet", + "-W default::PendingDeprecationWarning", f"--junitxml=system_{session.python}_sponge_log.xml", system_test_folder_path, *session.posargs, @@ -372,6 +375,7 @@ def prerelease(session): session.run( "py.test", "--quiet", + "-W default::PendingDeprecationWarning", f"--junitxml=prerelease_unit_{session.python}_sponge_log.xml", os.path.join("tests", "unit"), *session.posargs, @@ -380,6 +384,7 @@ def prerelease(session): session.run( "py.test", "--quiet", + "-W default::PendingDeprecationWarning", f"--junitxml=prerelease_system_{session.python}_sponge_log.xml", os.path.join("tests", "system"), *session.posargs, diff --git a/packages/pandas-gbq/pandas_gbq/__init__.py b/packages/pandas-gbq/pandas_gbq/__init__.py index 6d92bfa2190a..76c33d60adde 100644 --- a/packages/pandas-gbq/pandas_gbq/__init__.py +++ b/packages/pandas-gbq/pandas_gbq/__init__.py @@ -2,10 +2,24 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. +import warnings + from pandas_gbq import version as pandas_gbq_version +from . import _versions_helpers from .gbq import Context, context, read_gbq, to_gbq # noqa +sys_major, sys_minor, sys_micro = _versions_helpers.extract_runtime_version() +if sys_major == 3 and sys_minor in (7, 8): + warnings.warn( + "The python-bigquery library will stop supporting Python 3.7 " + "and Python 3.8 in a future major release expected in Q4 2024. " + f"Your Python version is {sys_major}.{sys_minor}.{sys_micro}. We " + "recommend that you update soon to ensure ongoing support. For " + "more details, see: [Google Cloud Client Libraries Supported Python Versions policy](https://cloud.google.com/python/docs/supported-python-versions)", + PendingDeprecationWarning, + ) + __version__ = pandas_gbq_version.__version__ __all__ = [ diff --git a/packages/pandas-gbq/pandas_gbq/_versions_helpers.py b/packages/pandas-gbq/pandas_gbq/_versions_helpers.py new file mode 100644 index 000000000000..37247c456d81 --- /dev/null +++ b/packages/pandas-gbq/pandas_gbq/_versions_helpers.py @@ -0,0 +1,32 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared helper functions for verifying versions of installed modules.""" + + +import sys +from typing import Tuple + + +def extract_runtime_version() -> Tuple[int, int, int]: + # Retrieve the version information + version_info = sys.version_info + + # Extract the major, minor, and micro components + major = version_info.major + minor = version_info.minor + micro = version_info.micro + + # Display the version number in a clear format + return major, minor, micro From f7f079553f9bab796abee117e4f21fbee59de972 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Mon, 14 Oct 2024 16:52:59 -0400 Subject: [PATCH 440/519] feat: adds the capability to include custom user agent string (#819) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * adds the capability to include customer user agent string * Adds deprecation warning * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * clarifies purpose and asert for two tests regarding old verbose arg * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * updates comment regarding verbosity * Update pandas_gbq/gbq.py --------- Co-authored-by: Owl Bot --- packages/pandas-gbq/pandas_gbq/gbq.py | 79 ++++++++++++++++++- packages/pandas-gbq/tests/unit/test_gbq.py | 20 ++++- packages/pandas-gbq/tests/unit/test_to_gbq.py | 25 ++++++ 3 files changed, 120 insertions(+), 4 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 06b6bbf228f6..b04ad131ddc7 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -267,6 +267,8 @@ def __init__( auth_redirect_uri=None, client_id=None, client_secret=None, + user_agent=None, + rfc9110_delimiter=False, ): global context from google.api_core.exceptions import ClientError, GoogleAPIError @@ -284,6 +286,8 @@ def __init__( self.auth_redirect_uri = auth_redirect_uri self.client_id = client_id self.client_secret = client_secret + self.user_agent = user_agent + self.rfc9110_delimiter = rfc9110_delimiter default_project = None @@ -337,11 +341,15 @@ def log_elapsed_seconds(self, prefix="Elapsed", postfix="s.", overlong=6): def get_client(self): import google.api_core.client_info - import pandas bigquery = FEATURES.bigquery_try_import() + + user_agent = create_user_agent( + user_agent=self.user_agent, rfc9110_delimiter=self.rfc9110_delimiter + ) + client_info = google.api_core.client_info.ClientInfo( - user_agent="pandas-{}".format(pandas.__version__) + user_agent=user_agent, ) return bigquery.Client( project=self.project_id, @@ -961,6 +969,8 @@ def to_gbq( auth_redirect_uri=None, client_id=None, client_secret=None, + user_agent=None, + rfc9110_delimiter=False, ): """Write a DataFrame to a Google BigQuery table. @@ -1072,6 +1082,13 @@ def to_gbq( client_secret : str The Client Secret associated with the Client ID for the Google Cloud Project the user is attempting to connect to. + user_agent : str + Custom user agent string used as a prefix to the pandas version. + rfc9110_delimiter : bool + Sets user agent delimiter to a hyphen or a slash. + Default is False, meaning a hyphen will be used. + + .. versionadded:: 0.23.3 """ _test_google_api_imports() @@ -1130,6 +1147,8 @@ def to_gbq( auth_redirect_uri=auth_redirect_uri, client_id=client_id, client_secret=client_secret, + user_agent=user_agent, + rfc9110_delimiter=rfc9110_delimiter, ) bqclient = connector.client @@ -1409,3 +1428,59 @@ def create(self, dataset_id): self.client.create_dataset(dataset) except self.http_error as ex: self.process_http_error(ex) + + +def create_user_agent( + user_agent: Optional[str] = None, rfc9110_delimiter: bool = False +) -> str: + """Creates a user agent string. + + The legacy format of our the user agent string was: `product-x.y.z` (where x, + y, and z are the major, minor, and micro version numbers). + + Users are able to prepend this string with their own user agent identifier + to render something similar to ` pandas-x.y.z`. + + The legacy format used a hyphen to separate the product from the product + version which differs slightly from the format recommended by RFC9110, which is: + `product/x.y.z`. To produce a user agent more in line with the RFC, set + rfc9110_delimiter to True. This setting does not depend on whether a + user_agent is also supplied. + + Reference: + https://www.rfc-editor.org/info/rfc9110 + + Args: + user_agent (Optional[str]): User agent string. + + rfc9110_delimiter (Optional[bool]): Sets delimiter to a hyphen or a slash. + Default is False, meaning a hyphen will be used. + + Returns (str): + Customized user agent string. + + Deprecation Warning: + In a future major release, the default delimiter will be changed to + a `/` in accordance with RFC9110. + """ + import pandas as pd + + if rfc9110_delimiter: + delimiter = "/" + else: + warnings.warn( + "In a future major release, the default delimiter will be " + "changed to a `/` in accordance with RFC9110.", + PendingDeprecationWarning, + stacklevel=2, + ) + delimiter = "-" + + identity = f"pandas{delimiter}{pd.__version__}" + + if user_agent is None: + user_agent = identity + else: + user_agent = f"{user_agent} {identity}" + + return user_agent diff --git a/packages/pandas-gbq/tests/unit/test_gbq.py b/packages/pandas-gbq/tests/unit/test_gbq.py index 92a09a3fe6af..755748202abd 100644 --- a/packages/pandas-gbq/tests/unit/test_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_gbq.py @@ -635,7 +635,15 @@ def test_read_gbq_wo_verbose_w_new_pandas_no_warnings(monkeypatch, recwarn): mock.PropertyMock(return_value=False), ) gbq.read_gbq("SELECT 1", project_id="my-project", dialect="standard") - assert len(recwarn) == 0 + # This test was intended to check for warnings about the deprecation of + # the argument `verbose` (which was removed from gbq (~v0.4.0) and + # pandas (~v0.23.0). (See https://github.com/googleapis/python-bigquery-pandas/pull/158/files) + # This test should not fail upon seeing a warning in regards to a pending + # deprecation related to rfc9110 delimiters. + # TODO this and related tests have likely outlived their usefulness, + # consider removing. + for warning in recwarn.list: + assert "delimiter" in str(warning.message) def test_read_gbq_with_old_bq_raises_importerror(monkeypatch): @@ -660,7 +668,15 @@ def test_read_gbq_with_verbose_old_pandas_no_warnings(monkeypatch, recwarn): dialect="standard", verbose=True, ) - assert len(recwarn) == 0 + # This test was intended to check for warnings about the deprecation of + # the argument `verbose` (which was removed from gbq (~v0.4.0) and + # pandas (~v0.23.0). (See https://github.com/googleapis/python-bigquery-pandas/pull/158/files) + # This test should not fail upon seeing a warning in regards to a pending + # deprecation related to rfc9110 delimiters. + # TODO this and related tests have likely outlived their usefulness, + # consider removing. + for warning in recwarn.list: + assert "delimiter" in str(warning.message) def test_read_gbq_with_private_raises_notimplmentederror(): diff --git a/packages/pandas-gbq/tests/unit/test_to_gbq.py b/packages/pandas-gbq/tests/unit/test_to_gbq.py index 23b7c9bd9389..60ea8025b3a4 100644 --- a/packages/pandas-gbq/tests/unit/test_to_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_to_gbq.py @@ -4,6 +4,7 @@ import google.api_core.exceptions import google.cloud.bigquery +import pandas as pd from pandas import DataFrame import pytest @@ -158,3 +159,27 @@ def test_to_gbq_with_if_exists_unknown(): project_id="myproj", if_exists="unknown", ) + + +@pytest.mark.parametrize( + "user_agent,rfc9110_delimiter,expected", + [ + ( + "test_user_agent/2.0.42", + False, + f"test_user_agent/2.0.42 pandas-{pd.__version__}", + ), + (None, False, f"pandas-{pd.__version__}"), + ( + "test_user_agent/2.0.42", + True, + f"test_user_agent/2.0.42 pandas/{pd.__version__}", + ), + (None, True, f"pandas/{pd.__version__}"), + ], +) +def test_create_user_agent(user_agent, rfc9110_delimiter, expected): + from pandas_gbq.gbq import create_user_agent + + result = create_user_agent(user_agent, rfc9110_delimiter) + assert result == expected From aee1bb374cc000bc22747b7160d2c435ce189c33 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 15 Oct 2024 12:43:19 -0500 Subject: [PATCH 441/519] chore(main): release 0.24.0 (#816) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- packages/pandas-gbq/CHANGELOG.md | 27 +++++++++++++++++++++++ packages/pandas-gbq/pandas_gbq/version.py | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index f5b132850134..00370b381c38 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,32 @@ # Changelog +## [0.24.0](https://github.com/googleapis/python-bigquery-pandas/compare/v0.23.2...v0.24.0) (2024-10-14) + + +### ⚠ BREAKING CHANGES + +* `to_gbq` loads naive (no timezone) columns to BigQuery DATETIME instead of TIMESTAMP ([#814](https://github.com/googleapis/python-bigquery-pandas/issues/814)) +* `to_gbq` loads object column containing bool values to BOOLEAN instead of STRING ([#814](https://github.com/googleapis/python-bigquery-pandas/issues/814)) +* `to_gbq` loads object column containing dictionary values to STRUCT instead of STRING ([#814](https://github.com/googleapis/python-bigquery-pandas/issues/814)) +* `to_gbq` loads `unit8` columns to BigQuery INT64 instead of STRING ([#814](https://github.com/googleapis/python-bigquery-pandas/issues/814)) + +### Features + +* Adds the capability to include custom user agent string ([#819](https://github.com/googleapis/python-bigquery-pandas/issues/819)) ([d43457b](https://github.com/googleapis/python-bigquery-pandas/commit/d43457b3838bdc135337cae47c56af397bb1d6d1)) + + +### Bug Fixes + +* `to_gbq` loads `unit8` columns to BigQuery INT64 instead of STRING ([#814](https://github.com/googleapis/python-bigquery-pandas/issues/814)) ([107bb40](https://github.com/googleapis/python-bigquery-pandas/commit/107bb40218b531be1a4f646b8fb0cea5bdfd8aee)) +* `to_gbq` loads naive (no timezone) columns to BigQuery DATETIME instead of TIMESTAMP ([#814](https://github.com/googleapis/python-bigquery-pandas/issues/814)) ([107bb40](https://github.com/googleapis/python-bigquery-pandas/commit/107bb40218b531be1a4f646b8fb0cea5bdfd8aee)) +* `to_gbq` loads object column containing bool values to BOOLEAN instead of STRING ([#814](https://github.com/googleapis/python-bigquery-pandas/issues/814)) ([107bb40](https://github.com/googleapis/python-bigquery-pandas/commit/107bb40218b531be1a4f646b8fb0cea5bdfd8aee)) +* `to_gbq` loads object column containing dictionary values to STRUCT instead of STRING ([#814](https://github.com/googleapis/python-bigquery-pandas/issues/814)) ([107bb40](https://github.com/googleapis/python-bigquery-pandas/commit/107bb40218b531be1a4f646b8fb0cea5bdfd8aee)) + + +### Dependencies + +* Min pyarrow is now 4.0.0 to support compliant nested types ([#814](https://github.com/googleapis/python-bigquery-pandas/issues/814)) ([107bb40](https://github.com/googleapis/python-bigquery-pandas/commit/107bb40218b531be1a4f646b8fb0cea5bdfd8aee)) + ## [0.23.2](https://github.com/googleapis/python-bigquery-pandas/compare/v0.23.1...v0.23.2) (2024-09-20) diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py index 73bc39aeed48..ec3eb08c0bc7 100644 --- a/packages/pandas-gbq/pandas_gbq/version.py +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -2,4 +2,4 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -__version__ = "0.23.2" +__version__ = "0.24.0" From 706d9bd82d52b3310f64509888f3f1d4ad4a4202 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 12 Nov 2024 18:38:41 +0800 Subject: [PATCH 442/519] build: use multiScm for Kokoro release builds (#823) Source-Link: https://github.com/googleapis/synthtool/commit/0da16589204e7f61911f64fcb30ac2d3b6e59b31 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:5cddfe2fb5019bbf78335bc55f15bc13e18354a56b3ff46e1834f8e540807f05 Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 +- .../pandas-gbq/.github/release-trigger.yml | 1 + .../.kokoro/docker/docs/requirements.txt | 42 +- packages/pandas-gbq/.kokoro/docs/common.cfg | 6 +- packages/pandas-gbq/.kokoro/release.sh | 2 +- .../pandas-gbq/.kokoro/release/common.cfg | 8 +- packages/pandas-gbq/.kokoro/requirements.txt | 610 +++++++++--------- .../.kokoro/samples/python3.13/common.cfg | 40 ++ .../.kokoro/samples/python3.13/continuous.cfg | 6 + .../samples/python3.13/periodic-head.cfg | 11 + .../.kokoro/samples/python3.13/periodic.cfg | 6 + .../.kokoro/samples/python3.13/presubmit.cfg | 6 + .../pandas-gbq/.kokoro/test-samples-impl.sh | 3 +- .../pandas-gbq/samples/snippets/noxfile.py | 2 +- 14 files changed, 392 insertions(+), 355 deletions(-) create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.13/common.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.13/continuous.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.13/periodic-head.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.13/periodic.cfg create mode 100644 packages/pandas-gbq/.kokoro/samples/python3.13/presubmit.cfg diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 597e0c3261ca..7672b49b6307 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:e8dcfd7cbfd8beac3a3ff8d3f3185287ea0625d859168cc80faccfc9a7a00455 -# created: 2024-09-16T21:04:09.091105552Z + digest: sha256:5cddfe2fb5019bbf78335bc55f15bc13e18354a56b3ff46e1834f8e540807f05 +# created: 2024-10-31T01:41:07.349286254Z diff --git a/packages/pandas-gbq/.github/release-trigger.yml b/packages/pandas-gbq/.github/release-trigger.yml index d4ca94189e16..4bb79e58eadf 100644 --- a/packages/pandas-gbq/.github/release-trigger.yml +++ b/packages/pandas-gbq/.github/release-trigger.yml @@ -1 +1,2 @@ enabled: true +multiScmName: diff --git a/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt b/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt index 7129c7715594..66eacc82f041 100644 --- a/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt +++ b/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt @@ -4,39 +4,39 @@ # # pip-compile --allow-unsafe --generate-hashes requirements.in # -argcomplete==3.4.0 \ - --hash=sha256:69a79e083a716173e5532e0fa3bef45f793f4e61096cf52b5a42c0211c8b8aa5 \ - --hash=sha256:c2abcdfe1be8ace47ba777d4fce319eb13bf8ad9dace8d085dcad6eded88057f +argcomplete==3.5.1 \ + --hash=sha256:1a1d148bdaa3e3b93454900163403df41448a248af01b6e849edc5ac08e6c363 \ + --hash=sha256:eb1ee355aa2557bd3d0145de7b06b2a45b0ce461e1e7813f5d066039ab4177b4 # via nox colorlog==6.8.2 \ --hash=sha256:3e3e079a41feb5a1b64f978b5ea4f46040a94f11f0e8bbb8261e3dbbeca64d44 \ --hash=sha256:4dcbb62368e2800cb3c5abd348da7e53f6c362dda502ec27c560b2e58a66bd33 # via nox -distlib==0.3.8 \ - --hash=sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784 \ - --hash=sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64 +distlib==0.3.9 \ + --hash=sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87 \ + --hash=sha256:a60f20dea646b8a33f3e7772f74dc0b2d0772d2837ee1342a00645c81edf9403 # via virtualenv -filelock==3.15.4 \ - --hash=sha256:2207938cbc1844345cb01a5a95524dae30f0ce089eba5b00378295a17e3e90cb \ - --hash=sha256:6ca1fffae96225dab4c6eaf1c4f4f28cd2568d3ec2a44e15a08520504de468e7 +filelock==3.16.1 \ + --hash=sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0 \ + --hash=sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435 # via virtualenv -nox==2024.4.15 \ - --hash=sha256:6492236efa15a460ecb98e7b67562a28b70da006ab0be164e8821177577c0565 \ - --hash=sha256:ecf6700199cdfa9e5ea0a41ff5e6ef4641d09508eda6edb89d9987864115817f +nox==2024.10.9 \ + --hash=sha256:1d36f309a0a2a853e9bccb76bbef6bb118ba92fa92674d15604ca99adeb29eab \ + --hash=sha256:7aa9dc8d1c27e9f45ab046ffd1c3b2c4f7c91755304769df231308849ebded95 # via -r requirements.in packaging==24.1 \ --hash=sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002 \ --hash=sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124 # via nox -platformdirs==4.2.2 \ - --hash=sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee \ - --hash=sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3 +platformdirs==4.3.6 \ + --hash=sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907 \ + --hash=sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb # via virtualenv -tomli==2.0.1 \ - --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \ - --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f +tomli==2.0.2 \ + --hash=sha256:2ebe24485c53d303f690b0ec092806a085f07af5a5aa1464f3931eec36caaa38 \ + --hash=sha256:d46d457a85337051c36524bc5349dd91b1877838e2979ac5ced3e710ed8a60ed # via nox -virtualenv==20.26.3 \ - --hash=sha256:4c43a2a236279d9ea36a0d76f98d84bd6ca94ac4e0f4a3b9d46d05e10fea542a \ - --hash=sha256:8cc4a31139e796e9a7de2cd5cf2489de1217193116a8fd42328f1bd65f434589 +virtualenv==20.26.6 \ + --hash=sha256:280aede09a2a5c317e409a00102e7077c6432c5a38f0ef938e643805a7ad2c48 \ + --hash=sha256:7345cc5b25405607a624d8418154577459c3e0277f5466dd79c49d5e492995f2 # via nox diff --git a/packages/pandas-gbq/.kokoro/docs/common.cfg b/packages/pandas-gbq/.kokoro/docs/common.cfg index 09818f6aa05b..f30e60a480b3 100644 --- a/packages/pandas-gbq/.kokoro/docs/common.cfg +++ b/packages/pandas-gbq/.kokoro/docs/common.cfg @@ -30,9 +30,9 @@ env_vars: { env_vars: { key: "V2_STAGING_BUCKET" - # Push non-cloud library docs to `docs-staging-v2-staging` instead of the + # Push non-cloud library docs to `docs-staging-v2-dev` instead of the # Cloud RAD bucket `docs-staging-v2` - value: "docs-staging-v2-staging" + value: "docs-staging-v2-dev" } # It will upload the docker image after successful builds. @@ -64,4 +64,4 @@ before_action { keyname: "docuploader_service_account" } } -} \ No newline at end of file +} diff --git a/packages/pandas-gbq/.kokoro/release.sh b/packages/pandas-gbq/.kokoro/release.sh index f7527caa4e35..1757fe26f43b 100755 --- a/packages/pandas-gbq/.kokoro/release.sh +++ b/packages/pandas-gbq/.kokoro/release.sh @@ -23,7 +23,7 @@ python3 -m releasetool publish-reporter-script > /tmp/publisher-script; source / export PYTHONUNBUFFERED=1 # Move into the package, build the distribution and upload. -TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google-cloud-pypi-token-keystore-2") +TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google-cloud-pypi-token-keystore-3") cd github/python-bigquery-pandas python3 setup.py sdist bdist_wheel twine upload --username __token__ --password "${TWINE_PASSWORD}" dist/* diff --git a/packages/pandas-gbq/.kokoro/release/common.cfg b/packages/pandas-gbq/.kokoro/release/common.cfg index 8ad3ddb2061e..e8cb847b6ba7 100644 --- a/packages/pandas-gbq/.kokoro/release/common.cfg +++ b/packages/pandas-gbq/.kokoro/release/common.cfg @@ -28,17 +28,11 @@ before_action { fetch_keystore { keystore_resource { keystore_config_id: 73713 - keyname: "google-cloud-pypi-token-keystore-2" + keyname: "google-cloud-pypi-token-keystore-3" } } } -# Tokens needed to report release status back to GitHub -env_vars: { - key: "SECRET_MANAGER_KEYS" - value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem" -} - # Store the packages we uploaded to PyPI. That way, we have a record of exactly # what we published, which we can use to generate SBOMs and attestations. action { diff --git a/packages/pandas-gbq/.kokoro/requirements.txt b/packages/pandas-gbq/.kokoro/requirements.txt index 9622baf0ba38..006d8ef931bf 100644 --- a/packages/pandas-gbq/.kokoro/requirements.txt +++ b/packages/pandas-gbq/.kokoro/requirements.txt @@ -4,79 +4,94 @@ # # pip-compile --allow-unsafe --generate-hashes requirements.in # -argcomplete==3.4.0 \ - --hash=sha256:69a79e083a716173e5532e0fa3bef45f793f4e61096cf52b5a42c0211c8b8aa5 \ - --hash=sha256:c2abcdfe1be8ace47ba777d4fce319eb13bf8ad9dace8d085dcad6eded88057f +argcomplete==3.5.1 \ + --hash=sha256:1a1d148bdaa3e3b93454900163403df41448a248af01b6e849edc5ac08e6c363 \ + --hash=sha256:eb1ee355aa2557bd3d0145de7b06b2a45b0ce461e1e7813f5d066039ab4177b4 # via nox -attrs==23.2.0 \ - --hash=sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30 \ - --hash=sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1 +attrs==24.2.0 \ + --hash=sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346 \ + --hash=sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2 # via gcp-releasetool backports-tarfile==1.2.0 \ --hash=sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34 \ --hash=sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991 # via jaraco-context -cachetools==5.3.3 \ - --hash=sha256:0abad1021d3f8325b2fc1d2e9c8b9c9d57b04c3932657a72465447332c24d945 \ - --hash=sha256:ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105 +cachetools==5.5.0 \ + --hash=sha256:02134e8439cdc2ffb62023ce1debca2944c3f289d66bb17ead3ab3dede74b292 \ + --hash=sha256:2cc24fb4cbe39633fb7badd9db9ca6295d766d9c2995f245725a46715d050f2a # via google-auth -certifi==2024.7.4 \ - --hash=sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b \ - --hash=sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90 +certifi==2024.8.30 \ + --hash=sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8 \ + --hash=sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9 # via requests -cffi==1.16.0 \ - --hash=sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc \ - --hash=sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a \ - --hash=sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417 \ - --hash=sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab \ - --hash=sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520 \ - --hash=sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36 \ - --hash=sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743 \ - --hash=sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8 \ - --hash=sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed \ - --hash=sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684 \ - --hash=sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56 \ - --hash=sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324 \ - --hash=sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d \ - --hash=sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235 \ - --hash=sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e \ - --hash=sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088 \ - --hash=sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000 \ - --hash=sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7 \ - --hash=sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e \ - --hash=sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673 \ - --hash=sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c \ - --hash=sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe \ - --hash=sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2 \ - --hash=sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098 \ - --hash=sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8 \ - --hash=sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a \ - --hash=sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0 \ - --hash=sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b \ - --hash=sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896 \ - --hash=sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e \ - --hash=sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9 \ - --hash=sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2 \ - --hash=sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b \ - --hash=sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6 \ - --hash=sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404 \ - --hash=sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f \ - --hash=sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0 \ - --hash=sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4 \ - --hash=sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc \ - --hash=sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936 \ - --hash=sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba \ - --hash=sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872 \ - --hash=sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb \ - --hash=sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614 \ - --hash=sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1 \ - --hash=sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d \ - --hash=sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969 \ - --hash=sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b \ - --hash=sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4 \ - --hash=sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627 \ - --hash=sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956 \ - --hash=sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357 +cffi==1.17.1 \ + --hash=sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8 \ + --hash=sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2 \ + --hash=sha256:0e2b1fac190ae3ebfe37b979cc1ce69c81f4e4fe5746bb401dca63a9062cdaf1 \ + --hash=sha256:0f048dcf80db46f0098ccac01132761580d28e28bc0f78ae0d58048063317e15 \ + --hash=sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36 \ + --hash=sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824 \ + --hash=sha256:1d599671f396c4723d016dbddb72fe8e0397082b0a77a4fab8028923bec050e8 \ + --hash=sha256:28b16024becceed8c6dfbc75629e27788d8a3f9030691a1dbf9821a128b22c36 \ + --hash=sha256:2bb1a08b8008b281856e5971307cc386a8e9c5b625ac297e853d36da6efe9c17 \ + --hash=sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf \ + --hash=sha256:31000ec67d4221a71bd3f67df918b1f88f676f1c3b535a7eb473255fdc0b83fc \ + --hash=sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3 \ + --hash=sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed \ + --hash=sha256:45398b671ac6d70e67da8e4224a065cec6a93541bb7aebe1b198a61b58c7b702 \ + --hash=sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1 \ + --hash=sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8 \ + --hash=sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903 \ + --hash=sha256:5da5719280082ac6bd9aa7becb3938dc9f9cbd57fac7d2871717b1feb0902ab6 \ + --hash=sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d \ + --hash=sha256:636062ea65bd0195bc012fea9321aca499c0504409f413dc88af450b57ffd03b \ + --hash=sha256:6883e737d7d9e4899a8a695e00ec36bd4e5e4f18fabe0aca0efe0a4b44cdb13e \ + --hash=sha256:6b8b4a92e1c65048ff98cfe1f735ef8f1ceb72e3d5f0c25fdb12087a23da22be \ + --hash=sha256:6f17be4345073b0a7b8ea599688f692ac3ef23ce28e5df79c04de519dbc4912c \ + --hash=sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683 \ + --hash=sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9 \ + --hash=sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c \ + --hash=sha256:7596d6620d3fa590f677e9ee430df2958d2d6d6de2feeae5b20e82c00b76fbf8 \ + --hash=sha256:78122be759c3f8a014ce010908ae03364d00a1f81ab5c7f4a7a5120607ea56e1 \ + --hash=sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4 \ + --hash=sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655 \ + --hash=sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67 \ + --hash=sha256:9755e4345d1ec879e3849e62222a18c7174d65a6a92d5b346b1863912168b595 \ + --hash=sha256:98e3969bcff97cae1b2def8ba499ea3d6f31ddfdb7635374834cf89a1a08ecf0 \ + --hash=sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65 \ + --hash=sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41 \ + --hash=sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6 \ + --hash=sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401 \ + --hash=sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6 \ + --hash=sha256:ad9413ccdeda48c5afdae7e4fa2192157e991ff761e7ab8fdd8926f40b160cc3 \ + --hash=sha256:b2ab587605f4ba0bf81dc0cb08a41bd1c0a5906bd59243d56bad7668a6fc6c16 \ + --hash=sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93 \ + --hash=sha256:c03e868a0b3bc35839ba98e74211ed2b05d2119be4e8a0f224fba9384f1fe02e \ + --hash=sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4 \ + --hash=sha256:c7eac2ef9b63c79431bc4b25f1cd649d7f061a28808cbc6c47b534bd789ef964 \ + --hash=sha256:c9c3d058ebabb74db66e431095118094d06abf53284d9c81f27300d0e0d8bc7c \ + --hash=sha256:ca74b8dbe6e8e8263c0ffd60277de77dcee6c837a3d0881d8c1ead7268c9e576 \ + --hash=sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0 \ + --hash=sha256:cdf5ce3acdfd1661132f2a9c19cac174758dc2352bfe37d98aa7512c6b7178b3 \ + --hash=sha256:d016c76bdd850f3c626af19b0542c9677ba156e4ee4fccfdd7848803533ef662 \ + --hash=sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3 \ + --hash=sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff \ + --hash=sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5 \ + --hash=sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd \ + --hash=sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f \ + --hash=sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5 \ + --hash=sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14 \ + --hash=sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d \ + --hash=sha256:e221cf152cff04059d011ee126477f0d9588303eb57e88923578ace7baad17f9 \ + --hash=sha256:e31ae45bc2e29f6b2abd0de1cc3b9d5205aa847cafaecb8af1476a609a2f6eb7 \ + --hash=sha256:edae79245293e15384b51f88b00613ba9f7198016a5948b5dddf4917d4d26382 \ + --hash=sha256:f1e22e8c4419538cb197e4dd60acc919d7696e5ef98ee4da4e01d3f8cfa4cc5a \ + --hash=sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e \ + --hash=sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a \ + --hash=sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4 \ + --hash=sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99 \ + --hash=sha256:f7f5baafcc48261359e14bcd6d9bff6d4b28d9103847c9e136694cb0501aef87 \ + --hash=sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b # via cryptography charset-normalizer==2.1.1 \ --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 \ @@ -97,72 +112,67 @@ colorlog==6.8.2 \ # via # gcp-docuploader # nox -cryptography==42.0.8 \ - --hash=sha256:013629ae70b40af70c9a7a5db40abe5d9054e6f4380e50ce769947b73bf3caad \ - --hash=sha256:2346b911eb349ab547076f47f2e035fc8ff2c02380a7cbbf8d87114fa0f1c583 \ - --hash=sha256:2f66d9cd9147ee495a8374a45ca445819f8929a3efcd2e3df6428e46c3cbb10b \ - --hash=sha256:2f88d197e66c65be5e42cd72e5c18afbfae3f741742070e3019ac8f4ac57262c \ - --hash=sha256:31f721658a29331f895a5a54e7e82075554ccfb8b163a18719d342f5ffe5ecb1 \ - --hash=sha256:343728aac38decfdeecf55ecab3264b015be68fc2816ca800db649607aeee648 \ - --hash=sha256:5226d5d21ab681f432a9c1cf8b658c0cb02533eece706b155e5fbd8a0cdd3949 \ - --hash=sha256:57080dee41209e556a9a4ce60d229244f7a66ef52750f813bfbe18959770cfba \ - --hash=sha256:5a94eccb2a81a309806027e1670a358b99b8fe8bfe9f8d329f27d72c094dde8c \ - --hash=sha256:6b7c4f03ce01afd3b76cf69a5455caa9cfa3de8c8f493e0d3ab7d20611c8dae9 \ - --hash=sha256:7016f837e15b0a1c119d27ecd89b3515f01f90a8615ed5e9427e30d9cdbfed3d \ - --hash=sha256:81884c4d096c272f00aeb1f11cf62ccd39763581645b0812e99a91505fa48e0c \ - --hash=sha256:81d8a521705787afe7a18d5bfb47ea9d9cc068206270aad0b96a725022e18d2e \ - --hash=sha256:8d09d05439ce7baa8e9e95b07ec5b6c886f548deb7e0f69ef25f64b3bce842f2 \ - --hash=sha256:961e61cefdcb06e0c6d7e3a1b22ebe8b996eb2bf50614e89384be54c48c6b63d \ - --hash=sha256:9c0c1716c8447ee7dbf08d6db2e5c41c688544c61074b54fc4564196f55c25a7 \ - --hash=sha256:a0608251135d0e03111152e41f0cc2392d1e74e35703960d4190b2e0f4ca9c70 \ - --hash=sha256:a0c5b2b0585b6af82d7e385f55a8bc568abff8923af147ee3c07bd8b42cda8b2 \ - --hash=sha256:ad803773e9df0b92e0a817d22fd8a3675493f690b96130a5e24f1b8fabbea9c7 \ - --hash=sha256:b297f90c5723d04bcc8265fc2a0f86d4ea2e0f7ab4b6994459548d3a6b992a14 \ - --hash=sha256:ba4f0a211697362e89ad822e667d8d340b4d8d55fae72cdd619389fb5912eefe \ - --hash=sha256:c4783183f7cb757b73b2ae9aed6599b96338eb957233c58ca8f49a49cc32fd5e \ - --hash=sha256:c9bb2ae11bfbab395bdd072985abde58ea9860ed84e59dbc0463a5d0159f5b71 \ - --hash=sha256:cafb92b2bc622cd1aa6a1dce4b93307792633f4c5fe1f46c6b97cf67073ec961 \ - --hash=sha256:d45b940883a03e19e944456a558b67a41160e367a719833c53de6911cabba2b7 \ - --hash=sha256:dc0fdf6787f37b1c6b08e6dfc892d9d068b5bdb671198c72072828b80bd5fe4c \ - --hash=sha256:dea567d1b0e8bc5764b9443858b673b734100c2871dc93163f58c46a97a83d28 \ - --hash=sha256:dec9b018df185f08483f294cae6ccac29e7a6e0678996587363dc352dc65c842 \ - --hash=sha256:e3ec3672626e1b9e55afd0df6d774ff0e953452886e06e0f1eb7eb0c832e8902 \ - --hash=sha256:e599b53fd95357d92304510fb7bda8523ed1f79ca98dce2f43c115950aa78801 \ - --hash=sha256:fa76fbb7596cc5839320000cdd5d0955313696d9511debab7ee7278fc8b5c84a \ - --hash=sha256:fff12c88a672ab9c9c1cf7b0c80e3ad9e2ebd9d828d955c126be4fd3e5578c9e +cryptography==43.0.1 \ + --hash=sha256:014f58110f53237ace6a408b5beb6c427b64e084eb451ef25a28308270086494 \ + --hash=sha256:1bbcce1a551e262dfbafb6e6252f1ae36a248e615ca44ba302df077a846a8806 \ + --hash=sha256:203e92a75716d8cfb491dc47c79e17d0d9207ccffcbcb35f598fbe463ae3444d \ + --hash=sha256:27e613d7077ac613e399270253259d9d53872aaf657471473ebfc9a52935c062 \ + --hash=sha256:2bd51274dcd59f09dd952afb696bf9c61a7a49dfc764c04dd33ef7a6b502a1e2 \ + --hash=sha256:38926c50cff6f533f8a2dae3d7f19541432610d114a70808f0926d5aaa7121e4 \ + --hash=sha256:511f4273808ab590912a93ddb4e3914dfd8a388fed883361b02dea3791f292e1 \ + --hash=sha256:58d4e9129985185a06d849aa6df265bdd5a74ca6e1b736a77959b498e0505b85 \ + --hash=sha256:5b43d1ea6b378b54a1dc99dd8a2b5be47658fe9a7ce0a58ff0b55f4b43ef2b84 \ + --hash=sha256:61ec41068b7b74268fa86e3e9e12b9f0c21fcf65434571dbb13d954bceb08042 \ + --hash=sha256:666ae11966643886c2987b3b721899d250855718d6d9ce41b521252a17985f4d \ + --hash=sha256:68aaecc4178e90719e95298515979814bda0cbada1256a4485414860bd7ab962 \ + --hash=sha256:7c05650fe8023c5ed0d46793d4b7d7e6cd9c04e68eabe5b0aeea836e37bdcec2 \ + --hash=sha256:80eda8b3e173f0f247f711eef62be51b599b5d425c429b5d4ca6a05e9e856baa \ + --hash=sha256:8385d98f6a3bf8bb2d65a73e17ed87a3ba84f6991c155691c51112075f9ffc5d \ + --hash=sha256:88cce104c36870d70c49c7c8fd22885875d950d9ee6ab54df2745f83ba0dc365 \ + --hash=sha256:9d3cdb25fa98afdd3d0892d132b8d7139e2c087da1712041f6b762e4f807cc96 \ + --hash=sha256:a575913fb06e05e6b4b814d7f7468c2c660e8bb16d8d5a1faf9b33ccc569dd47 \ + --hash=sha256:ac119bb76b9faa00f48128b7f5679e1d8d437365c5d26f1c2c3f0da4ce1b553d \ + --hash=sha256:c1332724be35d23a854994ff0b66530119500b6053d0bd3363265f7e5e77288d \ + --hash=sha256:d03a475165f3134f773d1388aeb19c2d25ba88b6a9733c5c590b9ff7bbfa2e0c \ + --hash=sha256:d75601ad10b059ec832e78823b348bfa1a59f6b8d545db3a24fd44362a1564cb \ + --hash=sha256:de41fd81a41e53267cb020bb3a7212861da53a7d39f863585d13ea11049cf277 \ + --hash=sha256:e710bf40870f4db63c3d7d929aa9e09e4e7ee219e703f949ec4073b4294f6172 \ + --hash=sha256:ea25acb556320250756e53f9e20a4177515f012c9eaea17eb7587a8c4d8ae034 \ + --hash=sha256:f98bf604c82c416bc829e490c700ca1553eafdf2912a91e23a79d97d9801372a \ + --hash=sha256:fba1007b3ef89946dbbb515aeeb41e30203b004f0b4b00e5e16078b518563289 # via # -r requirements.in # gcp-releasetool # secretstorage -distlib==0.3.8 \ - --hash=sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784 \ - --hash=sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64 +distlib==0.3.9 \ + --hash=sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87 \ + --hash=sha256:a60f20dea646b8a33f3e7772f74dc0b2d0772d2837ee1342a00645c81edf9403 # via virtualenv docutils==0.21.2 \ --hash=sha256:3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f \ --hash=sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2 # via readme-renderer -filelock==3.15.4 \ - --hash=sha256:2207938cbc1844345cb01a5a95524dae30f0ce089eba5b00378295a17e3e90cb \ - --hash=sha256:6ca1fffae96225dab4c6eaf1c4f4f28cd2568d3ec2a44e15a08520504de468e7 +filelock==3.16.1 \ + --hash=sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0 \ + --hash=sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435 # via virtualenv gcp-docuploader==0.6.5 \ --hash=sha256:30221d4ac3e5a2b9c69aa52fdbef68cc3f27d0e6d0d90e220fc024584b8d2318 \ --hash=sha256:b7458ef93f605b9d46a4bf3a8dc1755dad1f31d030c8679edf304e343b347eea # via -r requirements.in -gcp-releasetool==2.0.1 \ - --hash=sha256:34314a910c08e8911d9c965bd44f8f2185c4f556e737d719c33a41f6a610de96 \ - --hash=sha256:b0d5863c6a070702b10883d37c4bdfd74bf930fe417f36c0c965d3b7c779ae62 +gcp-releasetool==2.1.1 \ + --hash=sha256:25639269f4eae510094f9dbed9894977e1966933211eb155a451deebc3fc0b30 \ + --hash=sha256:845f4ded3d9bfe8cc7fdaad789e83f4ea014affa77785259a7ddac4b243e099e # via -r requirements.in -google-api-core==2.19.1 \ - --hash=sha256:f12a9b8309b5e21d92483bbd47ce2c445861ec7d269ef6784ecc0ea8c1fa6125 \ - --hash=sha256:f4695f1e3650b316a795108a76a1c416e6afb036199d1c1f1f110916df479ffd +google-api-core==2.21.0 \ + --hash=sha256:4a152fd11a9f774ea606388d423b68aa7e6d6a0ffe4c8266f74979613ec09f81 \ + --hash=sha256:6869eacb2a37720380ba5898312af79a4d30b8bca1548fb4093e0697dc4bdf5d # via # google-cloud-core # google-cloud-storage -google-auth==2.31.0 \ - --hash=sha256:042c4702efa9f7d3c48d3a69341c209381b125faa6dbf3ebe56bc7e40ae05c23 \ - --hash=sha256:87805c36970047247c8afe614d4e3af8eceafc1ebba0c679fe75ddd1d575e871 +google-auth==2.35.0 \ + --hash=sha256:25df55f327ef021de8be50bad0dfd4a916ad0de96da86cd05661c9297723ad3f \ + --hash=sha256:f4c64ed4e01e8e8b646ef34c018f8bf3338df0c8e37d8b3bba40e7f574a3278a # via # gcp-releasetool # google-api-core @@ -172,97 +182,56 @@ google-cloud-core==2.4.1 \ --hash=sha256:9b7749272a812bde58fff28868d0c5e2f585b82f37e09a1f6ed2d4d10f134073 \ --hash=sha256:a9e6a4422b9ac5c29f79a0ede9485473338e2ce78d91f2370c01e730eab22e61 # via google-cloud-storage -google-cloud-storage==2.17.0 \ - --hash=sha256:49378abff54ef656b52dca5ef0f2eba9aa83dc2b2c72c78714b03a1a95fe9388 \ - --hash=sha256:5b393bc766b7a3bc6f5407b9e665b2450d36282614b7945e570b3480a456d1e1 +google-cloud-storage==2.18.2 \ + --hash=sha256:97a4d45c368b7d401ed48c4fdfe86e1e1cb96401c9e199e419d289e2c0370166 \ + --hash=sha256:aaf7acd70cdad9f274d29332673fcab98708d0e1f4dceb5a5356aaef06af4d99 # via gcp-docuploader -google-crc32c==1.5.0 \ - --hash=sha256:024894d9d3cfbc5943f8f230e23950cd4906b2fe004c72e29b209420a1e6b05a \ - --hash=sha256:02c65b9817512edc6a4ae7c7e987fea799d2e0ee40c53ec573a692bee24de876 \ - --hash=sha256:02ebb8bf46c13e36998aeaad1de9b48f4caf545e91d14041270d9dca767b780c \ - --hash=sha256:07eb3c611ce363c51a933bf6bd7f8e3878a51d124acfc89452a75120bc436289 \ - --hash=sha256:1034d91442ead5a95b5aaef90dbfaca8633b0247d1e41621d1e9f9db88c36298 \ - --hash=sha256:116a7c3c616dd14a3de8c64a965828b197e5f2d121fedd2f8c5585c547e87b02 \ - --hash=sha256:19e0a019d2c4dcc5e598cd4a4bc7b008546b0358bd322537c74ad47a5386884f \ - --hash=sha256:1c7abdac90433b09bad6c43a43af253e688c9cfc1c86d332aed13f9a7c7f65e2 \ - --hash=sha256:1e986b206dae4476f41bcec1faa057851f3889503a70e1bdb2378d406223994a \ - --hash=sha256:272d3892a1e1a2dbc39cc5cde96834c236d5327e2122d3aaa19f6614531bb6eb \ - --hash=sha256:278d2ed7c16cfc075c91378c4f47924c0625f5fc84b2d50d921b18b7975bd210 \ - --hash=sha256:2ad40e31093a4af319dadf503b2467ccdc8f67c72e4bcba97f8c10cb078207b5 \ - --hash=sha256:2e920d506ec85eb4ba50cd4228c2bec05642894d4c73c59b3a2fe20346bd00ee \ - --hash=sha256:3359fc442a743e870f4588fcf5dcbc1bf929df1fad8fb9905cd94e5edb02e84c \ - --hash=sha256:37933ec6e693e51a5b07505bd05de57eee12f3e8c32b07da7e73669398e6630a \ - --hash=sha256:398af5e3ba9cf768787eef45c803ff9614cc3e22a5b2f7d7ae116df8b11e3314 \ - --hash=sha256:3b747a674c20a67343cb61d43fdd9207ce5da6a99f629c6e2541aa0e89215bcd \ - --hash=sha256:461665ff58895f508e2866824a47bdee72497b091c730071f2b7575d5762ab65 \ - --hash=sha256:4c6fdd4fccbec90cc8a01fc00773fcd5fa28db683c116ee3cb35cd5da9ef6c37 \ - --hash=sha256:5829b792bf5822fd0a6f6eb34c5f81dd074f01d570ed7f36aa101d6fc7a0a6e4 \ - --hash=sha256:596d1f98fc70232fcb6590c439f43b350cb762fb5d61ce7b0e9db4539654cc13 \ - --hash=sha256:5ae44e10a8e3407dbe138984f21e536583f2bba1be9491239f942c2464ac0894 \ - --hash=sha256:635f5d4dd18758a1fbd1049a8e8d2fee4ffed124462d837d1a02a0e009c3ab31 \ - --hash=sha256:64e52e2b3970bd891309c113b54cf0e4384762c934d5ae56e283f9a0afcd953e \ - --hash=sha256:66741ef4ee08ea0b2cc3c86916ab66b6aef03768525627fd6a1b34968b4e3709 \ - --hash=sha256:67b741654b851abafb7bc625b6d1cdd520a379074e64b6a128e3b688c3c04740 \ - --hash=sha256:6ac08d24c1f16bd2bf5eca8eaf8304812f44af5cfe5062006ec676e7e1d50afc \ - --hash=sha256:6f998db4e71b645350b9ac28a2167e6632c239963ca9da411523bb439c5c514d \ - --hash=sha256:72218785ce41b9cfd2fc1d6a017dc1ff7acfc4c17d01053265c41a2c0cc39b8c \ - --hash=sha256:74dea7751d98034887dbd821b7aae3e1d36eda111d6ca36c206c44478035709c \ - --hash=sha256:759ce4851a4bb15ecabae28f4d2e18983c244eddd767f560165563bf9aefbc8d \ - --hash=sha256:77e2fd3057c9d78e225fa0a2160f96b64a824de17840351b26825b0848022906 \ - --hash=sha256:7c074fece789b5034b9b1404a1f8208fc2d4c6ce9decdd16e8220c5a793e6f61 \ - --hash=sha256:7c42c70cd1d362284289c6273adda4c6af8039a8ae12dc451dcd61cdabb8ab57 \ - --hash=sha256:7f57f14606cd1dd0f0de396e1e53824c371e9544a822648cd76c034d209b559c \ - --hash=sha256:83c681c526a3439b5cf94f7420471705bbf96262f49a6fe546a6db5f687a3d4a \ - --hash=sha256:8485b340a6a9e76c62a7dce3c98e5f102c9219f4cfbf896a00cf48caf078d438 \ - --hash=sha256:84e6e8cd997930fc66d5bb4fde61e2b62ba19d62b7abd7a69920406f9ecca946 \ - --hash=sha256:89284716bc6a5a415d4eaa11b1726d2d60a0cd12aadf5439828353662ede9dd7 \ - --hash=sha256:8b87e1a59c38f275c0e3676fc2ab6d59eccecfd460be267ac360cc31f7bcde96 \ - --hash=sha256:8f24ed114432de109aa9fd317278518a5af2d31ac2ea6b952b2f7782b43da091 \ - --hash=sha256:98cb4d057f285bd80d8778ebc4fde6b4d509ac3f331758fb1528b733215443ae \ - --hash=sha256:998679bf62b7fb599d2878aa3ed06b9ce688b8974893e7223c60db155f26bd8d \ - --hash=sha256:9ba053c5f50430a3fcfd36f75aff9caeba0440b2d076afdb79a318d6ca245f88 \ - --hash=sha256:9c99616c853bb585301df6de07ca2cadad344fd1ada6d62bb30aec05219c45d2 \ - --hash=sha256:a1fd716e7a01f8e717490fbe2e431d2905ab8aa598b9b12f8d10abebb36b04dd \ - --hash=sha256:a2355cba1f4ad8b6988a4ca3feed5bff33f6af2d7f134852cf279c2aebfde541 \ - --hash=sha256:b1f8133c9a275df5613a451e73f36c2aea4fe13c5c8997e22cf355ebd7bd0728 \ - --hash=sha256:b8667b48e7a7ef66afba2c81e1094ef526388d35b873966d8a9a447974ed9178 \ - --hash=sha256:ba1eb1843304b1e5537e1fca632fa894d6f6deca8d6389636ee5b4797affb968 \ - --hash=sha256:be82c3c8cfb15b30f36768797a640e800513793d6ae1724aaaafe5bf86f8f346 \ - --hash=sha256:c02ec1c5856179f171e032a31d6f8bf84e5a75c45c33b2e20a3de353b266ebd8 \ - --hash=sha256:c672d99a345849301784604bfeaeba4db0c7aae50b95be04dd651fd2a7310b93 \ - --hash=sha256:c6c777a480337ac14f38564ac88ae82d4cd238bf293f0a22295b66eb89ffced7 \ - --hash=sha256:cae0274952c079886567f3f4f685bcaf5708f0a23a5f5216fdab71f81a6c0273 \ - --hash=sha256:cd67cf24a553339d5062eff51013780a00d6f97a39ca062781d06b3a73b15462 \ - --hash=sha256:d3515f198eaa2f0ed49f8819d5732d70698c3fa37384146079b3799b97667a94 \ - --hash=sha256:d5280312b9af0976231f9e317c20e4a61cd2f9629b7bfea6a693d1878a264ebd \ - --hash=sha256:de06adc872bcd8c2a4e0dc51250e9e65ef2ca91be023b9d13ebd67c2ba552e1e \ - --hash=sha256:e1674e4307fa3024fc897ca774e9c7562c957af85df55efe2988ed9056dc4e57 \ - --hash=sha256:e2096eddb4e7c7bdae4bd69ad364e55e07b8316653234a56552d9c988bd2d61b \ - --hash=sha256:e560628513ed34759456a416bf86b54b2476c59144a9138165c9a1575801d0d9 \ - --hash=sha256:edfedb64740750e1a3b16152620220f51d58ff1b4abceb339ca92e934775c27a \ - --hash=sha256:f13cae8cc389a440def0c8c52057f37359014ccbc9dc1f0827936bcd367c6100 \ - --hash=sha256:f314013e7dcd5cf45ab1945d92e713eec788166262ae8deb2cfacd53def27325 \ - --hash=sha256:f583edb943cf2e09c60441b910d6a20b4d9d626c75a36c8fcac01a6c96c01183 \ - --hash=sha256:fd8536e902db7e365f49e7d9029283403974ccf29b13fc7028b97e2295b33556 \ - --hash=sha256:fe70e325aa68fa4b5edf7d1a4b6f691eb04bbccac0ace68e34820d283b5f80d4 +google-crc32c==1.6.0 \ + --hash=sha256:05e2d8c9a2f853ff116db9706b4a27350587f341eda835f46db3c0a8c8ce2f24 \ + --hash=sha256:18e311c64008f1f1379158158bb3f0c8d72635b9eb4f9545f8cf990c5668e59d \ + --hash=sha256:236c87a46cdf06384f614e9092b82c05f81bd34b80248021f729396a78e55d7e \ + --hash=sha256:35834855408429cecf495cac67ccbab802de269e948e27478b1e47dfb6465e57 \ + --hash=sha256:386122eeaaa76951a8196310432c5b0ef3b53590ef4c317ec7588ec554fec5d2 \ + --hash=sha256:40b05ab32a5067525670880eb5d169529089a26fe35dce8891127aeddc1950e8 \ + --hash=sha256:48abd62ca76a2cbe034542ed1b6aee851b6f28aaca4e6551b5599b6f3ef175cc \ + --hash=sha256:50cf2a96da226dcbff8671233ecf37bf6e95de98b2a2ebadbfdf455e6d05df42 \ + --hash=sha256:51c4f54dd8c6dfeb58d1df5e4f7f97df8abf17a36626a217f169893d1d7f3e9f \ + --hash=sha256:5bcc90b34df28a4b38653c36bb5ada35671ad105c99cfe915fb5bed7ad6924aa \ + --hash=sha256:62f6d4a29fea082ac4a3c9be5e415218255cf11684ac6ef5488eea0c9132689b \ + --hash=sha256:6eceb6ad197656a1ff49ebfbbfa870678c75be4344feb35ac1edf694309413dc \ + --hash=sha256:7aec8e88a3583515f9e0957fe4f5f6d8d4997e36d0f61624e70469771584c760 \ + --hash=sha256:91ca8145b060679ec9176e6de4f89b07363d6805bd4760631ef254905503598d \ + --hash=sha256:a184243544811e4a50d345838a883733461e67578959ac59964e43cca2c791e7 \ + --hash=sha256:a9e4b426c3702f3cd23b933436487eb34e01e00327fac20c9aebb68ccf34117d \ + --hash=sha256:bb0966e1c50d0ef5bc743312cc730b533491d60585a9a08f897274e57c3f70e0 \ + --hash=sha256:bb8b3c75bd157010459b15222c3fd30577042a7060e29d42dabce449c087f2b3 \ + --hash=sha256:bd5e7d2445d1a958c266bfa5d04c39932dc54093fa391736dbfdb0f1929c1fb3 \ + --hash=sha256:c87d98c7c4a69066fd31701c4e10d178a648c2cac3452e62c6b24dc51f9fcc00 \ + --hash=sha256:d2952396dc604544ea7476b33fe87faedc24d666fb0c2d5ac971a2b9576ab871 \ + --hash=sha256:d8797406499f28b5ef791f339594b0b5fdedf54e203b5066675c406ba69d705c \ + --hash=sha256:d9e9913f7bd69e093b81da4535ce27af842e7bf371cde42d1ae9e9bd382dc0e9 \ + --hash=sha256:e2806553238cd076f0a55bddab37a532b53580e699ed8e5606d0de1f856b5205 \ + --hash=sha256:ebab974b1687509e5c973b5c4b8b146683e101e102e17a86bd196ecaa4d099fc \ + --hash=sha256:ed767bf4ba90104c1216b68111613f0d5926fb3780660ea1198fc469af410e9d \ + --hash=sha256:f7a1fc29803712f80879b0806cb83ab24ce62fc8daf0569f2204a0cfd7f68ed4 # via # google-cloud-storage # google-resumable-media -google-resumable-media==2.7.1 \ - --hash=sha256:103ebc4ba331ab1bfdac0250f8033627a2cd7cde09e7ccff9181e31ba4315b2c \ - --hash=sha256:eae451a7b2e2cdbaaa0fd2eb00cc8a1ee5e95e16b55597359cbc3d27d7d90e33 +google-resumable-media==2.7.2 \ + --hash=sha256:3ce7551e9fe6d99e9a126101d2536612bb73486721951e9562fee0f90c6ababa \ + --hash=sha256:5280aed4629f2b60b847b0d42f9857fd4935c11af266744df33d8074cae92fe0 # via google-cloud-storage -googleapis-common-protos==1.63.2 \ - --hash=sha256:27a2499c7e8aff199665b22741997e485eccc8645aa9176c7c988e6fae507945 \ - --hash=sha256:27c5abdffc4911f28101e635de1533fb4cfd2c37fbaa9174587c799fac90aa87 +googleapis-common-protos==1.65.0 \ + --hash=sha256:2972e6c496f435b92590fd54045060867f3fe9be2c82ab148fc8885035479a63 \ + --hash=sha256:334a29d07cddc3aa01dee4988f9afd9b2916ee2ff49d6b757155dc0d197852c0 # via google-api-core -idna==3.7 \ - --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \ - --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0 +idna==3.10 \ + --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \ + --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3 # via requests -importlib-metadata==8.0.0 \ - --hash=sha256:15584cf2b1bf449d98ff8a6ff1abef57bf20f3ac6454f431736cd3e660921b2f \ - --hash=sha256:188bd24e4c346d3f0a933f275c2fec67050326a856b9a359881d7c2a697e8812 +importlib-metadata==8.5.0 \ + --hash=sha256:45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b \ + --hash=sha256:71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7 # via # -r requirements.in # keyring @@ -271,13 +240,13 @@ jaraco-classes==3.4.0 \ --hash=sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd \ --hash=sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790 # via keyring -jaraco-context==5.3.0 \ - --hash=sha256:3e16388f7da43d384a1a7cd3452e72e14732ac9fe459678773a3608a812bf266 \ - --hash=sha256:c2f67165ce1f9be20f32f650f25d8edfc1646a8aeee48ae06fb35f90763576d2 +jaraco-context==6.0.1 \ + --hash=sha256:9bae4ea555cf0b14938dc0aee7c9f32ed303aa20a3b73e7dc80111628792d1b3 \ + --hash=sha256:f797fc481b490edb305122c9181830a3a5b76d84ef6d1aef2fb9b47ab956f9e4 # via keyring -jaraco-functools==4.0.1 \ - --hash=sha256:3b24ccb921d6b593bdceb56ce14799204f473976e2a9d4b15b04d0f2c2326664 \ - --hash=sha256:d33fa765374c0611b52f8b3a795f8900869aa88c84769d4d1746cd68fb28c3e8 +jaraco-functools==4.1.0 \ + --hash=sha256:70f7e0e2ae076498e212562325e805204fc092d7b4c17e0e86c959e249701a9d \ + --hash=sha256:ad159f13428bc4acbf5541ad6dec511f91573b90fba04df61dafa2a1231cf649 # via keyring jeepney==0.8.0 \ --hash=sha256:5efe48d255973902f6badc3ce55e2aa6c5c3b3bc642059ef3a91247bcfcc5806 \ @@ -289,9 +258,9 @@ jinja2==3.1.4 \ --hash=sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369 \ --hash=sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d # via gcp-releasetool -keyring==25.2.1 \ - --hash=sha256:2458681cdefc0dbc0b7eb6cf75d0b98e59f9ad9b2d4edd319d18f68bdca95e50 \ - --hash=sha256:daaffd42dbda25ddafb1ad5fec4024e5bbcfe424597ca1ca452b299861e49f1b +keyring==25.4.1 \ + --hash=sha256:5426f817cf7f6f007ba5ec722b1bcad95a75b27d780343772ad76b17cb47b0bf \ + --hash=sha256:b07ebc55f3e8ed86ac81dd31ef14e81ace9dd9c3d4b5d77a6e9a2016d0d71a1b # via # gcp-releasetool # twine @@ -299,75 +268,76 @@ markdown-it-py==3.0.0 \ --hash=sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1 \ --hash=sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb # via rich -markupsafe==2.1.5 \ - --hash=sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf \ - --hash=sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff \ - --hash=sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f \ - --hash=sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3 \ - --hash=sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532 \ - --hash=sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f \ - --hash=sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617 \ - --hash=sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df \ - --hash=sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4 \ - --hash=sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906 \ - --hash=sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f \ - --hash=sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4 \ - --hash=sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8 \ - --hash=sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371 \ - --hash=sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2 \ - --hash=sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465 \ - --hash=sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52 \ - --hash=sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6 \ - --hash=sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169 \ - --hash=sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad \ - --hash=sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2 \ - --hash=sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0 \ - --hash=sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029 \ - --hash=sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f \ - --hash=sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a \ - --hash=sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced \ - --hash=sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5 \ - --hash=sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c \ - --hash=sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf \ - --hash=sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9 \ - --hash=sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb \ - --hash=sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad \ - --hash=sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3 \ - --hash=sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1 \ - --hash=sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46 \ - --hash=sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc \ - --hash=sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a \ - --hash=sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee \ - --hash=sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900 \ - --hash=sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5 \ - --hash=sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea \ - --hash=sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f \ - --hash=sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5 \ - --hash=sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e \ - --hash=sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a \ - --hash=sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f \ - --hash=sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50 \ - --hash=sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a \ - --hash=sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b \ - --hash=sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4 \ - --hash=sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff \ - --hash=sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2 \ - --hash=sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46 \ - --hash=sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b \ - --hash=sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf \ - --hash=sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5 \ - --hash=sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5 \ - --hash=sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab \ - --hash=sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd \ - --hash=sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68 +markupsafe==3.0.1 \ + --hash=sha256:0778de17cff1acaeccc3ff30cd99a3fd5c50fc58ad3d6c0e0c4c58092b859396 \ + --hash=sha256:0f84af7e813784feb4d5e4ff7db633aba6c8ca64a833f61d8e4eade234ef0c38 \ + --hash=sha256:17b2aea42a7280db02ac644db1d634ad47dcc96faf38ab304fe26ba2680d359a \ + --hash=sha256:242d6860f1fd9191aef5fae22b51c5c19767f93fb9ead4d21924e0bcb17619d8 \ + --hash=sha256:244dbe463d5fb6d7ce161301a03a6fe744dac9072328ba9fc82289238582697b \ + --hash=sha256:26627785a54a947f6d7336ce5963569b5d75614619e75193bdb4e06e21d447ad \ + --hash=sha256:2a4b34a8d14649315c4bc26bbfa352663eb51d146e35eef231dd739d54a5430a \ + --hash=sha256:2ae99f31f47d849758a687102afdd05bd3d3ff7dbab0a8f1587981b58a76152a \ + --hash=sha256:312387403cd40699ab91d50735ea7a507b788091c416dd007eac54434aee51da \ + --hash=sha256:3341c043c37d78cc5ae6e3e305e988532b072329639007fd408a476642a89fd6 \ + --hash=sha256:33d1c36b90e570ba7785dacd1faaf091203d9942bc036118fab8110a401eb1a8 \ + --hash=sha256:3e683ee4f5d0fa2dde4db77ed8dd8a876686e3fc417655c2ece9a90576905344 \ + --hash=sha256:3ffb4a8e7d46ed96ae48805746755fadd0909fea2306f93d5d8233ba23dda12a \ + --hash=sha256:40621d60d0e58aa573b68ac5e2d6b20d44392878e0bfc159012a5787c4e35bc8 \ + --hash=sha256:40f1e10d51c92859765522cbd79c5c8989f40f0419614bcdc5015e7b6bf97fc5 \ + --hash=sha256:45d42d132cff577c92bfba536aefcfea7e26efb975bd455db4e6602f5c9f45e7 \ + --hash=sha256:48488d999ed50ba8d38c581d67e496f955821dc183883550a6fbc7f1aefdc170 \ + --hash=sha256:4935dd7883f1d50e2ffecca0aa33dc1946a94c8f3fdafb8df5c330e48f71b132 \ + --hash=sha256:4c2d64fdba74ad16138300815cfdc6ab2f4647e23ced81f59e940d7d4a1469d9 \ + --hash=sha256:4c8817557d0de9349109acb38b9dd570b03cc5014e8aabf1cbddc6e81005becd \ + --hash=sha256:4ffaaac913c3f7345579db4f33b0020db693f302ca5137f106060316761beea9 \ + --hash=sha256:5a4cb365cb49b750bdb60b846b0c0bc49ed62e59a76635095a179d440540c346 \ + --hash=sha256:62fada2c942702ef8952754abfc1a9f7658a4d5460fabe95ac7ec2cbe0d02abc \ + --hash=sha256:67c519635a4f64e495c50e3107d9b4075aec33634272b5db1cde839e07367589 \ + --hash=sha256:6a54c43d3ec4cf2a39f4387ad044221c66a376e58c0d0e971d47c475ba79c6b5 \ + --hash=sha256:7044312a928a66a4c2a22644147bc61a199c1709712069a344a3fb5cfcf16915 \ + --hash=sha256:730d86af59e0e43ce277bb83970530dd223bf7f2a838e086b50affa6ec5f9295 \ + --hash=sha256:800100d45176652ded796134277ecb13640c1a537cad3b8b53da45aa96330453 \ + --hash=sha256:80fcbf3add8790caddfab6764bde258b5d09aefbe9169c183f88a7410f0f6dea \ + --hash=sha256:82b5dba6eb1bcc29cc305a18a3c5365d2af06ee71b123216416f7e20d2a84e5b \ + --hash=sha256:852dc840f6d7c985603e60b5deaae1d89c56cb038b577f6b5b8c808c97580f1d \ + --hash=sha256:8ad4ad1429cd4f315f32ef263c1342166695fad76c100c5d979c45d5570ed58b \ + --hash=sha256:8ae369e84466aa70f3154ee23c1451fda10a8ee1b63923ce76667e3077f2b0c4 \ + --hash=sha256:93e8248d650e7e9d49e8251f883eed60ecbc0e8ffd6349e18550925e31bd029b \ + --hash=sha256:973a371a55ce9ed333a3a0f8e0bcfae9e0d637711534bcb11e130af2ab9334e7 \ + --hash=sha256:9ba25a71ebf05b9bb0e2ae99f8bc08a07ee8e98c612175087112656ca0f5c8bf \ + --hash=sha256:a10860e00ded1dd0a65b83e717af28845bb7bd16d8ace40fe5531491de76b79f \ + --hash=sha256:a4792d3b3a6dfafefdf8e937f14906a51bd27025a36f4b188728a73382231d91 \ + --hash=sha256:a7420ceda262dbb4b8d839a4ec63d61c261e4e77677ed7c66c99f4e7cb5030dd \ + --hash=sha256:ad91738f14eb8da0ff82f2acd0098b6257621410dcbd4df20aaa5b4233d75a50 \ + --hash=sha256:b6a387d61fe41cdf7ea95b38e9af11cfb1a63499af2759444b99185c4ab33f5b \ + --hash=sha256:b954093679d5750495725ea6f88409946d69cfb25ea7b4c846eef5044194f583 \ + --hash=sha256:bbde71a705f8e9e4c3e9e33db69341d040c827c7afa6789b14c6e16776074f5a \ + --hash=sha256:beeebf760a9c1f4c07ef6a53465e8cfa776ea6a2021eda0d0417ec41043fe984 \ + --hash=sha256:c91b394f7601438ff79a4b93d16be92f216adb57d813a78be4446fe0f6bc2d8c \ + --hash=sha256:c97ff7fedf56d86bae92fa0a646ce1a0ec7509a7578e1ed238731ba13aabcd1c \ + --hash=sha256:cb53e2a99df28eee3b5f4fea166020d3ef9116fdc5764bc5117486e6d1211b25 \ + --hash=sha256:cbf445eb5628981a80f54087f9acdbf84f9b7d862756110d172993b9a5ae81aa \ + --hash=sha256:d06b24c686a34c86c8c1fba923181eae6b10565e4d80bdd7bc1c8e2f11247aa4 \ + --hash=sha256:d98e66a24497637dd31ccab090b34392dddb1f2f811c4b4cd80c230205c074a3 \ + --hash=sha256:db15ce28e1e127a0013dfb8ac243a8e392db8c61eae113337536edb28bdc1f97 \ + --hash=sha256:db842712984e91707437461930e6011e60b39136c7331e971952bb30465bc1a1 \ + --hash=sha256:e24bfe89c6ac4c31792793ad9f861b8f6dc4546ac6dc8f1c9083c7c4f2b335cd \ + --hash=sha256:e81c52638315ff4ac1b533d427f50bc0afc746deb949210bc85f05d4f15fd772 \ + --hash=sha256:e9393357f19954248b00bed7c56f29a25c930593a77630c719653d51e7669c2a \ + --hash=sha256:ee3941769bd2522fe39222206f6dd97ae83c442a94c90f2b7a25d847d40f4729 \ + --hash=sha256:f31ae06f1328595d762c9a2bf29dafd8621c7d3adc130cbb46278079758779ca \ + --hash=sha256:f94190df587738280d544971500b9cafc9b950d32efcb1fba9ac10d84e6aa4e6 \ + --hash=sha256:fa7d686ed9883f3d664d39d5a8e74d3c5f63e603c2e3ff0abcba23eac6542635 \ + --hash=sha256:fb532dd9900381d2e8f48172ddc5a59db4c445a11b9fab40b3b786da40d3b56b \ + --hash=sha256:fe32482b37b4b00c7a52a07211b479653b7fe4f22b2e481b9a9b099d8a430f2f # via jinja2 mdurl==0.1.2 \ --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba # via markdown-it-py -more-itertools==10.3.0 \ - --hash=sha256:e5d93ef411224fbcef366a6e8ddc4c5781bc6359d43412a65dd5964e46111463 \ - --hash=sha256:ea6a02e24a9161e51faad17a8782b92a0df82c12c1c8886fec7f0c3fa1a1b320 +more-itertools==10.5.0 \ + --hash=sha256:037b0d3203ce90cca8ab1defbbdac29d5f993fc20131f3664dc8d6acfa872aef \ + --hash=sha256:5482bfef7849c25dc3c6dd53a6173ae4795da2a41a80faea6700d9f5846c5da6 # via # jaraco-classes # jaraco-functools @@ -389,9 +359,9 @@ nh3==0.2.18 \ --hash=sha256:de3ceed6e661954871d6cd78b410213bdcb136f79aafe22aa7182e028b8c7307 \ --hash=sha256:f0eca9ca8628dbb4e916ae2491d72957fdd35f7a5d326b7032a345f111ac07fe # via readme-renderer -nox==2024.4.15 \ - --hash=sha256:6492236efa15a460ecb98e7b67562a28b70da006ab0be164e8821177577c0565 \ - --hash=sha256:ecf6700199cdfa9e5ea0a41ff5e6ef4641d09508eda6edb89d9987864115817f +nox==2024.10.9 \ + --hash=sha256:1d36f309a0a2a853e9bccb76bbef6bb118ba92fa92674d15604ca99adeb29eab \ + --hash=sha256:7aa9dc8d1c27e9f45ab046ffd1c3b2c4f7c91755304769df231308849ebded95 # via -r requirements.in packaging==24.1 \ --hash=sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002 \ @@ -403,41 +373,41 @@ pkginfo==1.10.0 \ --hash=sha256:5df73835398d10db79f8eecd5cd86b1f6d29317589ea70796994d49399af6297 \ --hash=sha256:889a6da2ed7ffc58ab5b900d888ddce90bce912f2d2de1dc1c26f4cb9fe65097 # via twine -platformdirs==4.2.2 \ - --hash=sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee \ - --hash=sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3 +platformdirs==4.3.6 \ + --hash=sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907 \ + --hash=sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb # via virtualenv proto-plus==1.24.0 \ --hash=sha256:30b72a5ecafe4406b0d339db35b56c4059064e69227b8c3bda7462397f966445 \ --hash=sha256:402576830425e5f6ce4c2a6702400ac79897dab0b4343821aa5188b0fab81a12 # via google-api-core -protobuf==5.27.2 \ - --hash=sha256:0e341109c609749d501986b835f667c6e1e24531096cff9d34ae411595e26505 \ - --hash=sha256:176c12b1f1c880bf7a76d9f7c75822b6a2bc3db2d28baa4d300e8ce4cde7409b \ - --hash=sha256:354d84fac2b0d76062e9b3221f4abbbacdfd2a4d8af36bab0474f3a0bb30ab38 \ - --hash=sha256:4fadd8d83e1992eed0248bc50a4a6361dc31bcccc84388c54c86e530b7f58863 \ - --hash=sha256:54330f07e4949d09614707c48b06d1a22f8ffb5763c159efd5c0928326a91470 \ - --hash=sha256:610e700f02469c4a997e58e328cac6f305f649826853813177e6290416e846c6 \ - --hash=sha256:7fc3add9e6003e026da5fc9e59b131b8f22b428b991ccd53e2af8071687b4fce \ - --hash=sha256:9e8f199bf7f97bd7ecebffcae45ebf9527603549b2b562df0fbc6d4d688f14ca \ - --hash=sha256:a109916aaac42bff84702fb5187f3edadbc7c97fc2c99c5ff81dd15dcce0d1e5 \ - --hash=sha256:b848dbe1d57ed7c191dfc4ea64b8b004a3f9ece4bf4d0d80a367b76df20bf36e \ - --hash=sha256:f3ecdef226b9af856075f28227ff2c90ce3a594d092c39bee5513573f25e2714 +protobuf==5.28.2 \ + --hash=sha256:2c69461a7fcc8e24be697624c09a839976d82ae75062b11a0972e41fd2cd9132 \ + --hash=sha256:35cfcb15f213449af7ff6198d6eb5f739c37d7e4f1c09b5d0641babf2cc0c68f \ + --hash=sha256:52235802093bd8a2811abbe8bf0ab9c5f54cca0a751fdd3f6ac2a21438bffece \ + --hash=sha256:59379674ff119717404f7454647913787034f03fe7049cbef1d74a97bb4593f0 \ + --hash=sha256:5e8a95246d581eef20471b5d5ba010d55f66740942b95ba9b872d918c459452f \ + --hash=sha256:87317e9bcda04a32f2ee82089a204d3a2f0d3c8aeed16568c7daf4756e4f1fe0 \ + --hash=sha256:8ddc60bf374785fb7cb12510b267f59067fa10087325b8e1855b898a0d81d276 \ + --hash=sha256:a8b9403fc70764b08d2f593ce44f1d2920c5077bf7d311fefec999f8c40f78b7 \ + --hash=sha256:c0ea0123dac3399a2eeb1a1443d82b7afc9ff40241433296769f7da42d142ec3 \ + --hash=sha256:ca53faf29896c526863366a52a8f4d88e69cd04ec9571ed6082fa117fac3ab36 \ + --hash=sha256:eeea10f3dc0ac7e6b4933d32db20662902b4ab81bf28df12218aa389e9c2102d # via # gcp-docuploader # gcp-releasetool # google-api-core # googleapis-common-protos # proto-plus -pyasn1==0.6.0 \ - --hash=sha256:3a35ab2c4b5ef98e17dfdec8ab074046fbda76e281c5a706ccd82328cfc8f64c \ - --hash=sha256:cca4bb0f2df5504f02f6f8a775b6e416ff9b0b3b16f7ee80b5a3153d9b804473 +pyasn1==0.6.1 \ + --hash=sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629 \ + --hash=sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034 # via # pyasn1-modules # rsa -pyasn1-modules==0.4.0 \ - --hash=sha256:831dbcea1b177b28c9baddf4c6d1013c24c3accd14a1873fffaa6a2e905f17b6 \ - --hash=sha256:be04f15b66c206eed667e0bb5ab27e2b1855ea54a842e5037738099e8ca4ae0b +pyasn1-modules==0.4.1 \ + --hash=sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd \ + --hash=sha256:c28e2dbf9c06ad61c71a075c7e0f9fd0f1b0bb2d2ad4377f240d33ac2ab60a7c # via google-auth pycparser==2.22 \ --hash=sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6 \ @@ -449,9 +419,9 @@ pygments==2.18.0 \ # via # readme-renderer # rich -pyjwt==2.8.0 \ - --hash=sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de \ - --hash=sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320 +pyjwt==2.9.0 \ + --hash=sha256:3b02fb0f44517787776cf48f2ae25d8e14f300e6d7545a4315cee571a415e850 \ + --hash=sha256:7e1e5b56cc735432a7369cbfa0efe50fa113ebecdc04ae6922deba8b84582d0c # via gcp-releasetool pyperclip==1.9.0 \ --hash=sha256:b7de0142ddc81bfc5c7507eea19da920b92252b548b96186caf94a5e2527d310 @@ -481,9 +451,9 @@ rfc3986==2.0.0 \ --hash=sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd \ --hash=sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c # via twine -rich==13.7.1 \ - --hash=sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222 \ - --hash=sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432 +rich==13.9.2 \ + --hash=sha256:51a2c62057461aaf7152b4d611168f93a9fc73068f8ded2790f29fe2b5366d0c \ + --hash=sha256:8c82a3d3f8dcfe9e734771313e606b39d8247bb6b826e196f4914b333b743cf1 # via twine rsa==4.9 \ --hash=sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7 \ @@ -499,9 +469,9 @@ six==1.16.0 \ # via # gcp-docuploader # python-dateutil -tomli==2.0.1 \ - --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \ - --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f +tomli==2.0.2 \ + --hash=sha256:2ebe24485c53d303f690b0ec092806a085f07af5a5aa1464f3931eec36caaa38 \ + --hash=sha256:d46d457a85337051c36524bc5349dd91b1877838e2979ac5ced3e710ed8a60ed # via nox twine==5.1.1 \ --hash=sha256:215dbe7b4b94c2c50a7315c0275d2258399280fbb7d04182c7e55e24b5f93997 \ @@ -510,28 +480,30 @@ twine==5.1.1 \ typing-extensions==4.12.2 \ --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \ --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8 - # via -r requirements.in -urllib3==2.2.2 \ - --hash=sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472 \ - --hash=sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168 + # via + # -r requirements.in + # rich +urllib3==2.2.3 \ + --hash=sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac \ + --hash=sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9 # via # requests # twine -virtualenv==20.26.3 \ - --hash=sha256:4c43a2a236279d9ea36a0d76f98d84bd6ca94ac4e0f4a3b9d46d05e10fea542a \ - --hash=sha256:8cc4a31139e796e9a7de2cd5cf2489de1217193116a8fd42328f1bd65f434589 +virtualenv==20.26.6 \ + --hash=sha256:280aede09a2a5c317e409a00102e7077c6432c5a38f0ef938e643805a7ad2c48 \ + --hash=sha256:7345cc5b25405607a624d8418154577459c3e0277f5466dd79c49d5e492995f2 # via nox -wheel==0.43.0 \ - --hash=sha256:465ef92c69fa5c5da2d1cf8ac40559a8c940886afcef87dcf14b9470862f1d85 \ - --hash=sha256:55c570405f142630c6b9f72fe09d9b67cf1477fcf543ae5b8dcb1f5b7377da81 +wheel==0.44.0 \ + --hash=sha256:2376a90c98cc337d18623527a97c31797bd02bad0033d41547043a1cbfbe448f \ + --hash=sha256:a29c3f2817e95ab89aa4660681ad547c0e9547f20e75b0562fe7723c9a2a9d49 # via -r requirements.in -zipp==3.19.2 \ - --hash=sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19 \ - --hash=sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c +zipp==3.20.2 \ + --hash=sha256:a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350 \ + --hash=sha256:bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -setuptools==70.2.0 \ - --hash=sha256:b8b8060bb426838fbe942479c90296ce976249451118ef566a5a0b7d8b78fb05 \ - --hash=sha256:bd63e505105011b25c3c11f753f7e3b8465ea739efddaccef8f0efac2137bac1 +setuptools==75.1.0 \ + --hash=sha256:35ab7fd3bcd95e6b7fd704e4a1539513edad446c097797f2985e0e4b960772f2 \ + --hash=sha256:d59a21b17a275fb872a9c3dae73963160ae079f1049ed956880cd7c09b120538 # via -r requirements.in diff --git a/packages/pandas-gbq/.kokoro/samples/python3.13/common.cfg b/packages/pandas-gbq/.kokoro/samples/python3.13/common.cfg new file mode 100644 index 000000000000..37ccc958cbc0 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.13/common.cfg @@ -0,0 +1,40 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.13" +} + +# Declare build specific Cloud project. +env_vars: { + key: "BUILD_SPECIFIC_GCLOUD_PROJECT" + value: "python-docs-samples-tests-313" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-pandas/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-bigquery-pandas/.kokoro/trampoline_v2.sh" diff --git a/packages/pandas-gbq/.kokoro/samples/python3.13/continuous.cfg b/packages/pandas-gbq/.kokoro/samples/python3.13/continuous.cfg new file mode 100644 index 000000000000..a1c8d9759c88 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.13/continuous.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.13/periodic-head.cfg b/packages/pandas-gbq/.kokoro/samples/python3.13/periodic-head.cfg new file mode 100644 index 000000000000..98efde4dc99d --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.13/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-bigquery-pandas/.kokoro/test-samples-against-head.sh" +} diff --git a/packages/pandas-gbq/.kokoro/samples/python3.13/periodic.cfg b/packages/pandas-gbq/.kokoro/samples/python3.13/periodic.cfg new file mode 100644 index 000000000000..71cd1e597e38 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.13/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} diff --git a/packages/pandas-gbq/.kokoro/samples/python3.13/presubmit.cfg b/packages/pandas-gbq/.kokoro/samples/python3.13/presubmit.cfg new file mode 100644 index 000000000000..a1c8d9759c88 --- /dev/null +++ b/packages/pandas-gbq/.kokoro/samples/python3.13/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/test-samples-impl.sh b/packages/pandas-gbq/.kokoro/test-samples-impl.sh index 55910c8ba178..53e365bc4e79 100755 --- a/packages/pandas-gbq/.kokoro/test-samples-impl.sh +++ b/packages/pandas-gbq/.kokoro/test-samples-impl.sh @@ -33,7 +33,8 @@ export PYTHONUNBUFFERED=1 env | grep KOKORO # Install nox -python3.9 -m pip install --upgrade --quiet nox +# `virtualenv==20.26.6` is added for Python 3.7 compatibility +python3.9 -m pip install --upgrade --quiet nox virtualenv==20.26.6 # Use secrets acessor service account to get secrets if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then diff --git a/packages/pandas-gbq/samples/snippets/noxfile.py b/packages/pandas-gbq/samples/snippets/noxfile.py index c36d5f2d81f3..494639d2fa5e 100644 --- a/packages/pandas-gbq/samples/snippets/noxfile.py +++ b/packages/pandas-gbq/samples/snippets/noxfile.py @@ -88,7 +88,7 @@ def get_pytest_env_vars() -> Dict[str, str]: # DO NOT EDIT - automatically generated. # All versions used to test samples. -ALL_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] +ALL_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] # Any default versions that should be ignored. IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] From 49c0553bd32ef6e83a34fa3277e6fb8ee68abeb0 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 12 Nov 2024 18:56:11 +0800 Subject: [PATCH 443/519] chore(python): remove obsolete release scripts and config files (#825) Source-Link: https://github.com/googleapis/synthtool/commit/635751753776b1a7cabd4dcaa48013a96274372d Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:91d0075c6f2fd6a073a06168feee19fa2a8507692f2519a1dc7de3366d157e99 Co-authored-by: Owl Bot Co-authored-by: Lingqing Gan --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 ++-- packages/pandas-gbq/.github/release-trigger.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 7672b49b6307..8685d9e2d6e7 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:5cddfe2fb5019bbf78335bc55f15bc13e18354a56b3ff46e1834f8e540807f05 -# created: 2024-10-31T01:41:07.349286254Z + digest: sha256:91d0075c6f2fd6a073a06168feee19fa2a8507692f2519a1dc7de3366d157e99 +# created: 2024-11-11T16:13:09.302418532Z \ No newline at end of file diff --git a/packages/pandas-gbq/.github/release-trigger.yml b/packages/pandas-gbq/.github/release-trigger.yml index 4bb79e58eadf..6601e1508e28 100644 --- a/packages/pandas-gbq/.github/release-trigger.yml +++ b/packages/pandas-gbq/.github/release-trigger.yml @@ -1,2 +1,2 @@ enabled: true -multiScmName: +multiScmName: python-bigquery-pandas From ea32500afa2a393eab97b91eb4d457557ca584c9 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Tue, 12 Nov 2024 12:18:04 +0100 Subject: [PATCH 444/519] chore(deps): update all dependencies (#815) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update all dependencies * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot Co-authored-by: Lingqing Gan --- packages/pandas-gbq/samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 47c271255cb9..768b8b470933 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -2,5 +2,5 @@ google-cloud-bigquery-storage==2.26.0 google-cloud-bigquery==3.25.0 pandas-gbq==0.23.1 pandas===2.0.3; python_version == '3.8' -pandas==2.2.2; python_version >= '3.9' +pandas==2.2.3; python_version >= '3.9' pyarrow==17.0.0; python_version >= '3.8' From f0e524bc21ccdef6c8226df4b9c19d14aa02b41b Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 12 Nov 2024 21:26:15 +0800 Subject: [PATCH 445/519] chore(python): update dependencies in .kokoro/docker/docs (#827) Source-Link: https://github.com/googleapis/synthtool/commit/59171c8f83f3522ce186e4d110d27e772da4ba7a Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:2ed982f884312e4883e01b5ab8af8b6935f0216a5a2d82928d273081fc3be562 Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 ++-- .../.kokoro/docker/docs/requirements.txt | 20 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 8685d9e2d6e7..6301519a9a05 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:91d0075c6f2fd6a073a06168feee19fa2a8507692f2519a1dc7de3366d157e99 -# created: 2024-11-11T16:13:09.302418532Z \ No newline at end of file + digest: sha256:2ed982f884312e4883e01b5ab8af8b6935f0216a5a2d82928d273081fc3be562 +# created: 2024-11-12T12:09:45.821174897Z diff --git a/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt b/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt index 66eacc82f041..8bb0764594b1 100644 --- a/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt +++ b/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # pip-compile --allow-unsafe --generate-hashes requirements.in @@ -8,9 +8,9 @@ argcomplete==3.5.1 \ --hash=sha256:1a1d148bdaa3e3b93454900163403df41448a248af01b6e849edc5ac08e6c363 \ --hash=sha256:eb1ee355aa2557bd3d0145de7b06b2a45b0ce461e1e7813f5d066039ab4177b4 # via nox -colorlog==6.8.2 \ - --hash=sha256:3e3e079a41feb5a1b64f978b5ea4f46040a94f11f0e8bbb8261e3dbbeca64d44 \ - --hash=sha256:4dcbb62368e2800cb3c5abd348da7e53f6c362dda502ec27c560b2e58a66bd33 +colorlog==6.9.0 \ + --hash=sha256:5906e71acd67cb07a71e779c47c4bcb45fb8c2993eebe9e5adcd6a6f1b283eff \ + --hash=sha256:bfba54a1b93b94f54e1f4fe48395725a3d92fd2a4af702f6bd70946bdc0c6ac2 # via nox distlib==0.3.9 \ --hash=sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87 \ @@ -24,9 +24,9 @@ nox==2024.10.9 \ --hash=sha256:1d36f309a0a2a853e9bccb76bbef6bb118ba92fa92674d15604ca99adeb29eab \ --hash=sha256:7aa9dc8d1c27e9f45ab046ffd1c3b2c4f7c91755304769df231308849ebded95 # via -r requirements.in -packaging==24.1 \ - --hash=sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002 \ - --hash=sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124 +packaging==24.2 \ + --hash=sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759 \ + --hash=sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f # via nox platformdirs==4.3.6 \ --hash=sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907 \ @@ -36,7 +36,7 @@ tomli==2.0.2 \ --hash=sha256:2ebe24485c53d303f690b0ec092806a085f07af5a5aa1464f3931eec36caaa38 \ --hash=sha256:d46d457a85337051c36524bc5349dd91b1877838e2979ac5ced3e710ed8a60ed # via nox -virtualenv==20.26.6 \ - --hash=sha256:280aede09a2a5c317e409a00102e7077c6432c5a38f0ef938e643805a7ad2c48 \ - --hash=sha256:7345cc5b25405607a624d8418154577459c3e0277f5466dd79c49d5e492995f2 +virtualenv==20.27.1 \ + --hash=sha256:142c6be10212543b32c6c45d3d3893dff89112cc588b7d0879ae5a1ec03a47ba \ + --hash=sha256:f11f1b8a29525562925f745563bfd48b189450f61fb34c4f9cc79dd5aa32a1f4 # via nox From 491888b96f9ee01c5a27d528c7075eaf0daa1bc0 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Tue, 12 Nov 2024 18:11:11 +0100 Subject: [PATCH 446/519] chore(deps): update all dependencies (#826) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update all dependencies * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * Update samples/snippets/requirements.txt --------- Co-authored-by: Owl Bot Co-authored-by: Tim Sweña (Swast) --- packages/pandas-gbq/samples/snippets/requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 768b8b470933..39e05780c4be 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,6 +1,6 @@ -google-cloud-bigquery-storage==2.26.0 -google-cloud-bigquery==3.25.0 -pandas-gbq==0.23.1 +google-cloud-bigquery-storage==2.27.0 +google-cloud-bigquery==3.27.0 +pandas-gbq==0.24.0 pandas===2.0.3; python_version == '3.8' pandas==2.2.3; python_version >= '3.9' -pyarrow==17.0.0; python_version >= '3.8' +pyarrow==18.0.0; python_version >= '3.9' From cbb3cdbfef6ee4715f8f28f92c94465410a90cc4 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Mon, 2 Dec 2024 20:50:11 +0100 Subject: [PATCH 447/519] chore(deps): update all dependencies (#829) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update all dependencies * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot --- packages/pandas-gbq/samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 39e05780c4be..3ae34fd16835 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -3,4 +3,4 @@ google-cloud-bigquery==3.27.0 pandas-gbq==0.24.0 pandas===2.0.3; python_version == '3.8' pandas==2.2.3; python_version >= '3.9' -pyarrow==18.0.0; python_version >= '3.9' +pyarrow==18.1.0; python_version >= '3.9' From 4783b9f7c0c37eea3f3fee701dfee4ec2b314422 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Tue, 3 Dec 2024 19:10:52 +0100 Subject: [PATCH 448/519] chore(deps): update all dependencies (#830) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update all dependencies * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot --- packages/pandas-gbq/samples/snippets/requirements-test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements-test.txt b/packages/pandas-gbq/samples/snippets/requirements-test.txt index 96cc3163a065..e1f01f29fe81 100644 --- a/packages/pandas-gbq/samples/snippets/requirements-test.txt +++ b/packages/pandas-gbq/samples/snippets/requirements-test.txt @@ -1,2 +1,2 @@ google-cloud-testutils==1.4.0 -pytest==8.3.3 +pytest==8.3.4 From 7689b4fa7dc84e7bc0063e835abb8ebcbae0d86f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Swe=C3=B1a=20=28Swast=29?= Date: Wed, 11 Dec 2024 09:50:33 -0600 Subject: [PATCH 449/519] fix!: `to_gbq` uploads `ArrowDtype(pa.timestamp(...)` without timezone as `DATETIME` type (#832) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix!: `to_gbq` uploads `ArrowDtype(pa.timestamp(...)` without timezone as `DATETIME` type Release-As: 0.25.0 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot --- .../pandas_gbq/schema/pyarrow_to_bigquery.py | 9 +++++ .../unit/schema/test_pyarrow_to_bigquery.py | 36 +++++++++++++++++-- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/schema/pyarrow_to_bigquery.py b/packages/pandas-gbq/pandas_gbq/schema/pyarrow_to_bigquery.py index c63559ebff72..da1a1ce82d9a 100644 --- a/packages/pandas-gbq/pandas_gbq/schema/pyarrow_to_bigquery.py +++ b/packages/pandas-gbq/pandas_gbq/schema/pyarrow_to_bigquery.py @@ -38,6 +38,15 @@ def arrow_type_to_bigquery_field(name, type_) -> Optional[schema.SchemaField]: + # Since both TIMESTAMP/DATETIME use pyarrow.timestamp(...), we need to use + # a special case to disambiguate them. See: + # https://github.com/googleapis/python-bigquery-pandas/issues/450 + if pyarrow.types.is_timestamp(type_): + if type_.tz is None: + return schema.SchemaField(name, "DATETIME") + else: + return schema.SchemaField(name, "TIMESTAMP") + detected_type = _ARROW_SCALAR_IDS_TO_BQ.get(type_.id, None) if detected_type is not None: return schema.SchemaField(name, detected_type) diff --git a/packages/pandas-gbq/tests/unit/schema/test_pyarrow_to_bigquery.py b/packages/pandas-gbq/tests/unit/schema/test_pyarrow_to_bigquery.py index 9a20e3426ec5..4af0760fc1c5 100644 --- a/packages/pandas-gbq/tests/unit/schema/test_pyarrow_to_bigquery.py +++ b/packages/pandas-gbq/tests/unit/schema/test_pyarrow_to_bigquery.py @@ -2,13 +2,46 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. +from google.cloud import bigquery import pyarrow +import pytest from pandas_gbq.schema import pyarrow_to_bigquery +@pytest.mark.parametrize( + ( + "pyarrow_type", + "bigquery_type", + ), + ( + # All integer types should map to BigQuery INT64 (or INTEGER since + # SchemaField uses the legacy SQL names). See: + # https://github.com/googleapis/python-bigquery-pandas/issues/616 + (pyarrow.int8(), "INTEGER"), + (pyarrow.int16(), "INTEGER"), + (pyarrow.int32(), "INTEGER"), + (pyarrow.int64(), "INTEGER"), + (pyarrow.uint8(), "INTEGER"), + (pyarrow.uint16(), "INTEGER"), + (pyarrow.uint32(), "INTEGER"), + (pyarrow.uint64(), "INTEGER"), + # If there is no associated timezone, assume a naive (timezone-less) + # DATETIME. See: + # https://github.com/googleapis/python-bigquery-pandas/issues/450 + (pyarrow.timestamp("ns"), "DATETIME"), + (pyarrow.timestamp("ns", tz="UTC"), "TIMESTAMP"), + ), +) +def test_arrow_type_to_bigquery_field_scalar_types(pyarrow_type, bigquery_type): + field: bigquery.SchemaField = pyarrow_to_bigquery.arrow_type_to_bigquery_field( + "test_name", pyarrow_type + ) + assert field.name == "test_name" + assert field.field_type == bigquery_type + + def test_arrow_type_to_bigquery_field_unknown(): - # Default types should be picked at a higher layer. assert ( pyarrow_to_bigquery.arrow_type_to_bigquery_field("test_name", pyarrow.null()) is None @@ -16,7 +49,6 @@ def test_arrow_type_to_bigquery_field_unknown(): def test_arrow_type_to_bigquery_field_list_of_unknown(): - # Default types should be picked at a higher layer. assert ( pyarrow_to_bigquery.arrow_type_to_bigquery_field( "test_name", pyarrow.list_(pyarrow.null()) From 2ca4ae079b656e316e2fa2d7ee2d0bc87cff3af4 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 11 Dec 2024 10:23:59 -0600 Subject: [PATCH 450/519] chore(main): release 0.25.0 (#834) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- packages/pandas-gbq/CHANGELOG.md | 11 +++++++++++ packages/pandas-gbq/pandas_gbq/version.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index 00370b381c38..bcf55cd4dd52 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## [0.25.0](https://github.com/googleapis/python-bigquery-pandas/compare/v0.24.0...v0.25.0) (2024-12-11) + + +### ⚠ BREAKING CHANGES + +* to_gbq uploads ArrowDtype(pa.timestamp(...) without timezone as DATETIME type ([#832](https://github.com/googleapis/python-bigquery-pandas/issues/832)) + +### Bug Fixes + +* To_gbq uploads ArrowDtype(pa.timestamp(...) without timezone as DATETIME type ([#832](https://github.com/googleapis/python-bigquery-pandas/issues/832)) ([2104b71](https://github.com/googleapis/python-bigquery-pandas/commit/2104b71a8ac1513a49b6e8bb73636d6b2f363d0e)) + ## [0.24.0](https://github.com/googleapis/python-bigquery-pandas/compare/v0.23.2...v0.24.0) (2024-10-14) diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py index ec3eb08c0bc7..478b813610b4 100644 --- a/packages/pandas-gbq/pandas_gbq/version.py +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -2,4 +2,4 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -__version__ = "0.24.0" +__version__ = "0.25.0" From 853cd4067db28050014ce5406cfd9776db4d1931 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Swe=C3=B1a=20=28Swast=29?= Date: Thu, 12 Dec 2024 15:38:50 -0600 Subject: [PATCH 451/519] feat: `to_gbq` fails with `TypeError` if passing in a bigframes DataFrame object (#833) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: `to_gbq` fails with `TypeError` if passing in a bigframes DataFrame object * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot --- packages/pandas-gbq/pandas_gbq/gbq.py | 8 ++++++++ packages/pandas-gbq/tests/unit/test_to_gbq.py | 19 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index b04ad131ddc7..feffd8587088 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -1091,6 +1091,14 @@ def to_gbq( .. versionadded:: 0.23.3 """ + # If we get a bigframes.pandas.DataFrame object, it may be possible to use + # the code paths here, but it could potentially be quite expensive because + # of the queries involved in type detection. It would be safer just to + # fail early if there are bigframes-y methods available. + # https://github.com/googleapis/python-bigquery-pandas/issues/824 + if hasattr(dataframe, "to_pandas") and hasattr(dataframe, "to_gbq"): + raise TypeError(f"Expected a pandas.DataFrame, but got {repr(type(dataframe))}") + _test_google_api_imports() from google.api_core import exceptions as google_exceptions diff --git a/packages/pandas-gbq/tests/unit/test_to_gbq.py b/packages/pandas-gbq/tests/unit/test_to_gbq.py index 60ea8025b3a4..f4012dc84353 100644 --- a/packages/pandas-gbq/tests/unit/test_to_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_to_gbq.py @@ -11,6 +11,16 @@ from pandas_gbq import gbq +class FakeDataFrame: + """A fake bigframes DataFrame to avoid depending on bigframes.""" + + def to_gbq(self): + """Fake to_gbq() to mimic a bigframes object.""" + + def to_pandas(self): + """Fake to_pandas() to mimic a bigframes object.""" + + @pytest.fixture def expected_load_method(mock_bigquery_client): return mock_bigquery_client.load_table_from_dataframe @@ -66,6 +76,15 @@ def test_to_gbq_load_method_translates_exception( expected_load_method.assert_called_once() +def test_to_gbq_with_bigframes_raises_typeerror(): + dataframe = FakeDataFrame() + + with pytest.raises( + TypeError, match=r"Expected a pandas.DataFrame, but got .+FakeDataFrame" + ): + gbq.to_gbq(dataframe, "my_dataset.my_table", project_id="myproj") + + def test_to_gbq_with_if_exists_append(mock_bigquery_client, expected_load_method): from google.cloud.bigquery import SchemaField From c2f553829537fe85141a99d060dc06d4bf055aed Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Wed, 18 Dec 2024 02:41:50 +0800 Subject: [PATCH 452/519] chore(python): update dependencies in .kokoro/docker/docs (#841) Source-Link: https://github.com/googleapis/synthtool/commit/e808c98e1ab7eec3df2a95a05331619f7001daef Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:8e3e7e18255c22d1489258d0374c901c01f9c4fd77a12088670cd73d580aa737 Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 +- .../.kokoro/docker/docs/requirements.txt | 52 +++++++++++++++---- 2 files changed, 43 insertions(+), 13 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 6301519a9a05..26306af66f81 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:2ed982f884312e4883e01b5ab8af8b6935f0216a5a2d82928d273081fc3be562 -# created: 2024-11-12T12:09:45.821174897Z + digest: sha256:8e3e7e18255c22d1489258d0374c901c01f9c4fd77a12088670cd73d580aa737 +# created: 2024-12-17T00:59:58.625514486Z diff --git a/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt b/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt index 8bb0764594b1..f99a5c4aac7f 100644 --- a/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt +++ b/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt @@ -2,11 +2,11 @@ # This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile --allow-unsafe --generate-hashes requirements.in +# pip-compile --allow-unsafe --generate-hashes synthtool/gcp/templates/python_library/.kokoro/docker/docs/requirements.in # -argcomplete==3.5.1 \ - --hash=sha256:1a1d148bdaa3e3b93454900163403df41448a248af01b6e849edc5ac08e6c363 \ - --hash=sha256:eb1ee355aa2557bd3d0145de7b06b2a45b0ce461e1e7813f5d066039ab4177b4 +argcomplete==3.5.2 \ + --hash=sha256:036d020d79048a5d525bc63880d7a4b8d1668566b8a76daf1144c0bbe0f63472 \ + --hash=sha256:23146ed7ac4403b70bd6026402468942ceba34a6732255b9edf5b7354f68a6bb # via nox colorlog==6.9.0 \ --hash=sha256:5906e71acd67cb07a71e779c47c4bcb45fb8c2993eebe9e5adcd6a6f1b283eff \ @@ -23,7 +23,7 @@ filelock==3.16.1 \ nox==2024.10.9 \ --hash=sha256:1d36f309a0a2a853e9bccb76bbef6bb118ba92fa92674d15604ca99adeb29eab \ --hash=sha256:7aa9dc8d1c27e9f45ab046ffd1c3b2c4f7c91755304769df231308849ebded95 - # via -r requirements.in + # via -r synthtool/gcp/templates/python_library/.kokoro/docker/docs/requirements.in packaging==24.2 \ --hash=sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759 \ --hash=sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f @@ -32,11 +32,41 @@ platformdirs==4.3.6 \ --hash=sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907 \ --hash=sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb # via virtualenv -tomli==2.0.2 \ - --hash=sha256:2ebe24485c53d303f690b0ec092806a085f07af5a5aa1464f3931eec36caaa38 \ - --hash=sha256:d46d457a85337051c36524bc5349dd91b1877838e2979ac5ced3e710ed8a60ed +tomli==2.2.1 \ + --hash=sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6 \ + --hash=sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd \ + --hash=sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c \ + --hash=sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b \ + --hash=sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8 \ + --hash=sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6 \ + --hash=sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77 \ + --hash=sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff \ + --hash=sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea \ + --hash=sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192 \ + --hash=sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249 \ + --hash=sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee \ + --hash=sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4 \ + --hash=sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98 \ + --hash=sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8 \ + --hash=sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4 \ + --hash=sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281 \ + --hash=sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744 \ + --hash=sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69 \ + --hash=sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13 \ + --hash=sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140 \ + --hash=sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e \ + --hash=sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e \ + --hash=sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc \ + --hash=sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff \ + --hash=sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec \ + --hash=sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2 \ + --hash=sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222 \ + --hash=sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106 \ + --hash=sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272 \ + --hash=sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a \ + --hash=sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7 # via nox -virtualenv==20.27.1 \ - --hash=sha256:142c6be10212543b32c6c45d3d3893dff89112cc588b7d0879ae5a1ec03a47ba \ - --hash=sha256:f11f1b8a29525562925f745563bfd48b189450f61fb34c4f9cc79dd5aa32a1f4 +virtualenv==20.28.0 \ + --hash=sha256:23eae1b4516ecd610481eda647f3a7c09aea295055337331bb4e6892ecce47b0 \ + --hash=sha256:2c9c3262bb8e7b87ea801d715fae4495e6032450c71d2309be9550e7364049aa # via nox From 6586cf928f6c2aa6f39765047a6f111b893994a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Swe=C3=B1a=20=28Swast=29?= Date: Thu, 19 Dec 2024 11:22:03 -0600 Subject: [PATCH 453/519] fix: `to_gbq` uses `default_type` for ambiguous array types and struct field types (#838) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: `to_gbq` uses `default_type` for ambiguous array types and struct field types * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * fix arrow list(null) case too * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * lint * Update pandas_gbq/schema/pandas_to_bigquery.py Co-authored-by: Chalmer Lowe * Update pandas_gbq/schema/pandas_to_bigquery.py Co-authored-by: Chalmer Lowe * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * remove redundant string check * Apply suggestions from code review Co-authored-by: Chalmer Lowe * add docstrings and a few more test cases * use python 3.10 for docs github action --------- Co-authored-by: Owl Bot Co-authored-by: Chalmer Lowe --- .../pandas-gbq/.github/workflows/docs.yml | 2 +- packages/pandas-gbq/owlbot.py | 1 + .../pandas_gbq/schema/pandas_to_bigquery.py | 111 +++++++++++++++--- .../pandas_gbq/schema/pyarrow_to_bigquery.py | 61 +++++++++- .../unit/schema/test_pandas_to_bigquery.py | 49 ++++++-- .../unit/schema/test_pyarrow_to_bigquery.py | 18 ++- packages/pandas-gbq/tests/unit/test_schema.py | 51 +++++++- 7 files changed, 244 insertions(+), 49 deletions(-) diff --git a/packages/pandas-gbq/.github/workflows/docs.yml b/packages/pandas-gbq/.github/workflows/docs.yml index 698fbc5c94da..2833fe98fff0 100644 --- a/packages/pandas-gbq/.github/workflows/docs.yml +++ b/packages/pandas-gbq/.github/workflows/docs.yml @@ -12,7 +12,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: "3.9" + python-version: "3.10" - name: Install nox run: | python -m pip install --upgrade setuptools pip wheel diff --git a/packages/pandas-gbq/owlbot.py b/packages/pandas-gbq/owlbot.py index 190298a64c00..e50b9e9e6586 100644 --- a/packages/pandas-gbq/owlbot.py +++ b/packages/pandas-gbq/owlbot.py @@ -57,6 +57,7 @@ "noxfile.py", "README.rst", # exclude this file as we have an alternate prerelease.cfg + ".github/workflows/docs.yml", ".kokoro/presubmit/prerelease-deps.cfg", ".kokoro/presubmit/presubmit.cfg", ], diff --git a/packages/pandas-gbq/pandas_gbq/schema/pandas_to_bigquery.py b/packages/pandas-gbq/pandas_gbq/schema/pandas_to_bigquery.py index 5a979a128e7b..5afae356024e 100644 --- a/packages/pandas-gbq/pandas_gbq/schema/pandas_to_bigquery.py +++ b/packages/pandas-gbq/pandas_gbq/schema/pandas_to_bigquery.py @@ -4,7 +4,7 @@ import collections.abc import datetime -from typing import Optional, Tuple +from typing import Any, Optional, Tuple import warnings import db_dtypes @@ -28,14 +28,21 @@ # `docs/source/writing.rst`. _PANDAS_DTYPE_TO_BQ = { "bool": "BOOLEAN", + "boolean": "BOOLEAN", "datetime64[ns, UTC]": "TIMESTAMP", + "datetime64[us, UTC]": "TIMESTAMP", "datetime64[ns]": "DATETIME", + "datetime64[us]": "DATETIME", "float32": "FLOAT", "float64": "FLOAT", "int8": "INTEGER", "int16": "INTEGER", "int32": "INTEGER", "int64": "INTEGER", + "Int8": "INTEGER", + "Int16": "INTEGER", + "Int32": "INTEGER", + "Int64": "INTEGER", "uint8": "INTEGER", "uint16": "INTEGER", "uint32": "INTEGER", @@ -103,7 +110,7 @@ def dataframe_to_bigquery_fields( # Try to automatically determine the type based on a few rows of the data. values = dataframe.reset_index()[column] - bq_field = values_to_bigquery_field(column, values) + bq_field = values_to_bigquery_field(column, values, default_type=default_type) if bq_field: bq_schema_out.append(bq_field) @@ -114,7 +121,9 @@ def dataframe_to_bigquery_fields( arrow_value = pyarrow.array(values) bq_field = ( pandas_gbq.schema.pyarrow_to_bigquery.arrow_type_to_bigquery_field( - column, arrow_value.type + column, + arrow_value.type, + default_type=default_type, ) ) @@ -151,6 +160,19 @@ def dataframe_to_bigquery_fields( def dtype_to_bigquery_field(name, dtype) -> Optional[schema.SchemaField]: + """Infers the BigQuery schema field type from a pandas dtype. + + Args: + name (str): + Name of the column/field. + dtype: + A pandas / numpy dtype object. + + Returns: + Optional[schema.SchemaField]: + The schema field, or None if a type cannot be inferred, such as if + it is ambiguous like the object dtype. + """ bq_type = _PANDAS_DTYPE_TO_BQ.get(dtype.name) if bq_type is not None: @@ -164,9 +186,44 @@ def dtype_to_bigquery_field(name, dtype) -> Optional[schema.SchemaField]: return None -def value_to_bigquery_field(name, value) -> Optional[schema.SchemaField]: - if isinstance(value, str): - return schema.SchemaField(name, "STRING") +def value_to_bigquery_field( + name: str, value: Any, default_type: Optional[str] = None +) -> Optional[schema.SchemaField]: + """Infers the BigQuery schema field type from a single value. + + Args: + name: + The name of the field. + value: + The value to infer the type from. If None, the default type is used + if available. + default_type: + The default field type. Defaults to None. + + Returns: + The schema field, or None if a type cannot be inferred. + """ + + # Set the SchemaField datatype to the given default_type if the value + # being assessed is None. + if value is None: + return schema.SchemaField(name, default_type) + + # Map from Python types to BigQuery types. This isn't super exhaustive + # because we rely more on pyarrow, which can check more than one value to + # determine the type. + type_mapping = { + str: "STRING", + } + + # geopandas and shapely are optional dependencies, so only check if those + # are installed. + if _BaseGeometry is not None: + type_mapping[_BaseGeometry] = "GEOGRAPHY" + + for type_, bq_type in type_mapping.items(): + if isinstance(value, type_): + return schema.SchemaField(name, bq_type) # For timezone-naive datetimes, the later pyarrow conversion to try and # learn the type add a timezone to such datetimes, causing them to be @@ -182,35 +239,51 @@ def value_to_bigquery_field(name, value) -> Optional[schema.SchemaField]: else: return schema.SchemaField(name, "DATETIME") - if _BaseGeometry is not None and isinstance(value, _BaseGeometry): - return schema.SchemaField(name, "GEOGRAPHY") - return None -def values_to_bigquery_field(name, values) -> Optional[schema.SchemaField]: +def values_to_bigquery_field( + name: str, values: Any, default_type: str = "STRING" +) -> Optional[schema.SchemaField]: + """Infers the BigQuery schema field type from a list of values. + + This function iterates through the given values to determine the + corresponding schema field type. + + Args: + name: + The name of the field. + values: + An iterable of values to infer the type from. If all the values + are None or the iterable is empty, the function returns None. + default_type: + The default field type to use if a specific type cannot be + determined from the values. Defaults to "STRING". + + Returns: + The schema field, or None if a type cannot be inferred. + """ value = pandas_gbq.core.pandas.first_valid(values) - # All NULL, type not determinable. + # All values came back as NULL, thus type not determinable by this method. + # Return None so we can try other methods. if value is None: return None - field = value_to_bigquery_field(name, value) - if field is not None: + field = value_to_bigquery_field(name, value, default_type=default_type) + if field: return field - if isinstance(value, str): - return schema.SchemaField(name, "STRING") - - # Check plain ARRAY values here. Let STRUCT get determined by pyarrow, - # which can examine more values to determine all keys. + # Check plain ARRAY values here. Exclude mapping types to let STRUCT get + # determined by pyarrow, which can examine more values to determine all + # keys. if isinstance(value, collections.abc.Iterable) and not isinstance( value, collections.abc.Mapping ): # It could be that this value contains all None or is empty, so get the # first non-None value we can find. valid_item = pandas_gbq.core.pandas.first_array_valid(values) - field = value_to_bigquery_field(name, valid_item) + field = value_to_bigquery_field(name, valid_item, default_type=default_type) if field is not None: return schema.SchemaField(name, field.field_type, mode="REPEATED") diff --git a/packages/pandas-gbq/pandas_gbq/schema/pyarrow_to_bigquery.py b/packages/pandas-gbq/pandas_gbq/schema/pyarrow_to_bigquery.py index da1a1ce82d9a..91677f9d89bf 100644 --- a/packages/pandas-gbq/pandas_gbq/schema/pyarrow_to_bigquery.py +++ b/packages/pandas-gbq/pandas_gbq/schema/pyarrow_to_bigquery.py @@ -37,7 +37,31 @@ } -def arrow_type_to_bigquery_field(name, type_) -> Optional[schema.SchemaField]: +def arrow_type_to_bigquery_field( + name, type_, default_type="STRING" +) -> Optional[schema.SchemaField]: + """Infers the BigQuery schema field type from an arrow type. + + Args: + name (str): + Name of the column/field. + type_: + A pyarrow type object. + + Returns: + Optional[schema.SchemaField]: + The schema field, or None if a type cannot be inferred, such as if + it is a type that doesn't have a clear mapping in BigQuery. + + null() are assumed to be the ``default_type``, since there are no + values that contradict that. + """ + # If a sub-field is the null type, then assume it's the default type, as + # that's the best we can do. + # https://github.com/googleapis/python-bigquery-pandas/issues/836 + if pyarrow.types.is_null(type_): + return schema.SchemaField(name, default_type) + # Since both TIMESTAMP/DATETIME use pyarrow.timestamp(...), we need to use # a special case to disambiguate them. See: # https://github.com/googleapis/python-bigquery-pandas/issues/450 @@ -52,22 +76,49 @@ def arrow_type_to_bigquery_field(name, type_) -> Optional[schema.SchemaField]: return schema.SchemaField(name, detected_type) if pyarrow.types.is_list(type_): - return arrow_list_type_to_bigquery(name, type_) + return arrow_list_type_to_bigquery(name, type_, default_type=default_type) if pyarrow.types.is_struct(type_): inner_fields: list[pyarrow.Field] = [] struct_type = cast(pyarrow.StructType, type_) for field_index in range(struct_type.num_fields): field = struct_type[field_index] - inner_fields.append(arrow_type_to_bigquery_field(field.name, field.type)) + inner_fields.append( + arrow_type_to_bigquery_field( + field.name, field.type, default_type=default_type + ) + ) return schema.SchemaField(name, "RECORD", fields=inner_fields) return None -def arrow_list_type_to_bigquery(name, type_) -> Optional[schema.SchemaField]: - inner_field = arrow_type_to_bigquery_field(name, type_.value_type) +def arrow_list_type_to_bigquery( + name, type_, default_type="STRING" +) -> Optional[schema.SchemaField]: + """Infers the BigQuery schema field type from an arrow list type. + + Args: + name (str): + Name of the column/field. + type_: + A pyarrow type object. + + Returns: + Optional[schema.SchemaField]: + The schema field, or None if a type cannot be inferred, such as if + it is a type that doesn't have a clear mapping in BigQuery. + + null() are assumed to be the ``default_type``, since there are no + values that contradict that. + """ + inner_field = arrow_type_to_bigquery_field( + name, type_.value_type, default_type=default_type + ) + + # If this is None, it means we got some type that we can't cleanly map to + # a BigQuery type, so bubble that status up. if inner_field is None: return None diff --git a/packages/pandas-gbq/tests/unit/schema/test_pandas_to_bigquery.py b/packages/pandas-gbq/tests/unit/schema/test_pandas_to_bigquery.py index 924ce1eecab2..777c38256294 100644 --- a/packages/pandas-gbq/tests/unit/schema/test_pandas_to_bigquery.py +++ b/packages/pandas-gbq/tests/unit/schema/test_pandas_to_bigquery.py @@ -21,13 +21,34 @@ def module_under_test(): def test_dataframe_to_bigquery_fields_w_named_index(module_under_test): df_data = collections.OrderedDict( [ + ("str_index", ["a", "b"]), ("str_column", ["hello", "world"]), ("int_column", [42, 8]), + ("nullable_int_column", pandas.Series([42, None], dtype="Int64")), + ("uint_column", pandas.Series([7, 13], dtype="uint8")), ("bool_column", [True, False]), + ("boolean_column", pandas.Series([True, None], dtype="boolean")), + ( + "datetime_column", + [ + datetime.datetime(1999, 12, 31, 23, 59, 59, 999999), + datetime.datetime(2000, 1, 1, 0, 0, 0), + ], + ), + ( + "timestamp_column", + [ + datetime.datetime( + 1999, 12, 31, 23, 59, 59, 999999, tzinfo=datetime.timezone.utc + ), + datetime.datetime( + 2000, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc + ), + ], + ), ] ) - index = pandas.Index(["a", "b"], name="str_index") - dataframe = pandas.DataFrame(df_data, index=index) + dataframe = pandas.DataFrame(df_data).set_index("str_index", drop=True) returned_schema = module_under_test.dataframe_to_bigquery_fields( dataframe, [], index=True @@ -37,7 +58,12 @@ def test_dataframe_to_bigquery_fields_w_named_index(module_under_test): schema.SchemaField("str_index", "STRING", "NULLABLE"), schema.SchemaField("str_column", "STRING", "NULLABLE"), schema.SchemaField("int_column", "INTEGER", "NULLABLE"), + schema.SchemaField("nullable_int_column", "INTEGER", "NULLABLE"), + schema.SchemaField("uint_column", "INTEGER", "NULLABLE"), schema.SchemaField("bool_column", "BOOLEAN", "NULLABLE"), + schema.SchemaField("boolean_column", "BOOLEAN", "NULLABLE"), + schema.SchemaField("datetime_column", "DATETIME", "NULLABLE"), + schema.SchemaField("timestamp_column", "TIMESTAMP", "NULLABLE"), ) assert returned_schema == expected_schema @@ -45,19 +71,24 @@ def test_dataframe_to_bigquery_fields_w_named_index(module_under_test): def test_dataframe_to_bigquery_fields_w_multiindex(module_under_test): df_data = collections.OrderedDict( [ + ("str_index", ["a", "a"]), + ("int_index", [0, 0]), + ( + "dt_index", + [ + datetime.datetime(1999, 12, 31, 23, 59, 59, 999999), + datetime.datetime(2000, 1, 1, 0, 0, 0), + ], + ), ("str_column", ["hello", "world"]), ("int_column", [42, 8]), ("bool_column", [True, False]), ] ) - index = pandas.MultiIndex.from_tuples( - [ - ("a", 0, datetime.datetime(1999, 12, 31, 23, 59, 59, 999999)), - ("a", 0, datetime.datetime(2000, 1, 1, 0, 0, 0)), - ], - names=["str_index", "int_index", "dt_index"], + dataframe = pandas.DataFrame(df_data).set_index( + ["str_index", "int_index", "dt_index"], + drop=True, ) - dataframe = pandas.DataFrame(df_data, index=index) returned_schema = module_under_test.dataframe_to_bigquery_fields( dataframe, [], index=True diff --git a/packages/pandas-gbq/tests/unit/schema/test_pyarrow_to_bigquery.py b/packages/pandas-gbq/tests/unit/schema/test_pyarrow_to_bigquery.py index 4af0760fc1c5..dc5504f99e12 100644 --- a/packages/pandas-gbq/tests/unit/schema/test_pyarrow_to_bigquery.py +++ b/packages/pandas-gbq/tests/unit/schema/test_pyarrow_to_bigquery.py @@ -42,16 +42,14 @@ def test_arrow_type_to_bigquery_field_scalar_types(pyarrow_type, bigquery_type): def test_arrow_type_to_bigquery_field_unknown(): - assert ( - pyarrow_to_bigquery.arrow_type_to_bigquery_field("test_name", pyarrow.null()) - is None - ) + assert pyarrow_to_bigquery.arrow_type_to_bigquery_field( + "test_name", pyarrow.null(), default_type="DEFAULT_TYPE" + ) == bigquery.SchemaField("test_name", "DEFAULT_TYPE") def test_arrow_type_to_bigquery_field_list_of_unknown(): - assert ( - pyarrow_to_bigquery.arrow_type_to_bigquery_field( - "test_name", pyarrow.list_(pyarrow.null()) - ) - is None - ) + assert pyarrow_to_bigquery.arrow_type_to_bigquery_field( + "test_name", + pyarrow.list_(pyarrow.null()), + default_type="DEFAULT_TYPE", + ) == bigquery.SchemaField("test_name", "DEFAULT_TYPE", mode="REPEATED") diff --git a/packages/pandas-gbq/tests/unit/test_schema.py b/packages/pandas-gbq/tests/unit/test_schema.py index 48e8862a8174..0da16bafd9b6 100644 --- a/packages/pandas-gbq/tests/unit/test_schema.py +++ b/packages/pandas-gbq/tests/unit/test_schema.py @@ -70,7 +70,7 @@ def test_schema_is_subset_fails_if_not_subset(): [ pytest.param( pandas.DataFrame(data={"col1": [object()]}), - {"fields": [{"name": "col1", "type": "STRING"}]}, + {"fields": [{"name": "col1", "type": "DEFAULT_TYPE"}]}, id="default-type-fails-pyarrow-conversion", ), ( @@ -182,13 +182,15 @@ def test_schema_is_subset_fails_if_not_subset(): else "object", ), "list_of_struct": pandas.Series( - [[], [{"test": "abc"}], []], + [[], [{"test": 123.0}], []], dtype=pandas.ArrowDtype( - pyarrow.list_(pyarrow.struct([("test", pyarrow.string())])) + pyarrow.list_(pyarrow.struct([("test", pyarrow.float64())])) ) if hasattr(pandas, "ArrowDtype") else "object", ), + "list_of_unknown": [[], [], []], + "list_of_null": [[None, None], [None], [None, None]], } ), { @@ -200,17 +202,56 @@ def test_schema_is_subset_fails_if_not_subset(): "type": "RECORD", "mode": "REPEATED", "fields": [ - {"name": "test", "type": "STRING", "mode": "NULLABLE"}, + {"name": "test", "type": "FLOAT", "mode": "NULLABLE"}, ], }, + # Use DEFAULT_TYPE because there are no values to detect a type. + { + "name": "list_of_unknown", + "type": "DEFAULT_TYPE", + "mode": "REPEATED", + }, + { + "name": "list_of_null", + "type": "DEFAULT_TYPE", + "mode": "REPEATED", + }, ], }, id="array", ), + pytest.param( + # If a struct contains only nulls in a sub-field, use the default + # type for subfields without a type we can determine. + # https://github.com/googleapis/python-bigquery-pandas/issues/836 + pandas.DataFrame( + { + "id": [0, 1], + "positions": [{"state": None}, {"state": None}], + }, + ), + { + "fields": [ + {"name": "id", "type": "INTEGER"}, + { + "name": "positions", + "type": "RECORD", + "fields": [ + { + "name": "state", + "type": "DEFAULT_TYPE", + "mode": "NULLABLE", + }, + ], + }, + ], + }, + id="issue832-null-struct-field", + ), ], ) def test_generate_bq_schema(dataframe, expected_schema): - schema = pandas_gbq.gbq._generate_bq_schema(dataframe) + schema = pandas_gbq.gbq._generate_bq_schema(dataframe, default_type="DEFAULT_TYPE") # NULLABLE is the default mode. for field in expected_schema["fields"]: From da1213bfb9dc1d4fb9a91f39f501465d415f30a8 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Thu, 19 Dec 2024 11:48:05 -0600 Subject: [PATCH 454/519] chore(main): release 0.26.0 (#837) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- packages/pandas-gbq/CHANGELOG.md | 12 ++++++++++++ packages/pandas-gbq/pandas_gbq/version.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index bcf55cd4dd52..41b4c8df27d3 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## [0.26.0](https://github.com/googleapis/python-bigquery-pandas/compare/v0.25.0...v0.26.0) (2024-12-19) + + +### Features + +* `to_gbq` fails with `TypeError` if passing in a bigframes DataFrame object ([#833](https://github.com/googleapis/python-bigquery-pandas/issues/833)) ([5004d08](https://github.com/googleapis/python-bigquery-pandas/commit/5004d08c6af93471686ccb319c69cd38c7893042)) + + +### Bug Fixes + +* `to_gbq` uses `default_type` for ambiguous array types and struct field types ([#838](https://github.com/googleapis/python-bigquery-pandas/issues/838)) ([cf1aadd](https://github.com/googleapis/python-bigquery-pandas/commit/cf1aadd48165617768fecff91e68941255148dbd)) + ## [0.25.0](https://github.com/googleapis/python-bigquery-pandas/compare/v0.24.0...v0.25.0) (2024-12-11) diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py index 478b813610b4..0c8dab154f65 100644 --- a/packages/pandas-gbq/pandas_gbq/version.py +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -2,4 +2,4 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -__version__ = "0.25.0" +__version__ = "0.26.0" From 1fcb689e7aef46a96972036e6cab4416e605cd7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Swe=C3=B1a=20=28Swast=29?= Date: Mon, 6 Jan 2025 10:49:34 -0600 Subject: [PATCH 455/519] fix: ensure BIGNUMERIC type is used if scale > 9 in Decimal values (#844) --- .../pandas_gbq/schema/pyarrow_to_bigquery.py | 7 +++++ .../unit/schema/test_pandas_to_bigquery.py | 26 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/packages/pandas-gbq/pandas_gbq/schema/pyarrow_to_bigquery.py b/packages/pandas-gbq/pandas_gbq/schema/pyarrow_to_bigquery.py index 91677f9d89bf..d917499fe079 100644 --- a/packages/pandas-gbq/pandas_gbq/schema/pyarrow_to_bigquery.py +++ b/packages/pandas-gbq/pandas_gbq/schema/pyarrow_to_bigquery.py @@ -72,6 +72,13 @@ def arrow_type_to_bigquery_field( return schema.SchemaField(name, "TIMESTAMP") detected_type = _ARROW_SCALAR_IDS_TO_BQ.get(type_.id, None) + + # We need a special case for values that might fit in Arrow decimal128 but + # not with the scale/precision that is used in BigQuery's NUMERIC type. + # See: https://github.com/googleapis/python-bigquery/issues/1650 + if detected_type == "NUMERIC" and type_.scale > 9: + detected_type = "BIGNUMERIC" + if detected_type is not None: return schema.SchemaField(name, detected_type) diff --git a/packages/pandas-gbq/tests/unit/schema/test_pandas_to_bigquery.py b/packages/pandas-gbq/tests/unit/schema/test_pandas_to_bigquery.py index 777c38256294..f3c4410b6d21 100644 --- a/packages/pandas-gbq/tests/unit/schema/test_pandas_to_bigquery.py +++ b/packages/pandas-gbq/tests/unit/schema/test_pandas_to_bigquery.py @@ -4,6 +4,7 @@ import collections import datetime +import decimal import operator from google.cloud.bigquery import schema @@ -46,6 +47,29 @@ def test_dataframe_to_bigquery_fields_w_named_index(module_under_test): ), ], ), + # Need to fallback to Arrow to avoid data loss and disambiguate + # NUMERIC from BIGNUMERIC. We don't want to pick too small of a + # type and lose precision. See: + # https://github.com/googleapis/python-bigquery/issues/1650 + # + ( + "bignumeric_column", + [ + # Start with a lower precision Decimal to make sure we + # aren't trying to determine the type from just one value. + decimal.Decimal("1.25"), + decimal.Decimal("0.1234567891"), + ], + ), + ( + "numeric_column", + [ + # Minimum value greater than 0 that can be handled: 1e-9 + # https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#numeric_types + decimal.Decimal("0.000000001"), + decimal.Decimal("-0.000000001"), + ], + ), ] ) dataframe = pandas.DataFrame(df_data).set_index("str_index", drop=True) @@ -64,6 +88,8 @@ def test_dataframe_to_bigquery_fields_w_named_index(module_under_test): schema.SchemaField("boolean_column", "BOOLEAN", "NULLABLE"), schema.SchemaField("datetime_column", "DATETIME", "NULLABLE"), schema.SchemaField("timestamp_column", "TIMESTAMP", "NULLABLE"), + schema.SchemaField("bignumeric_column", "BIGNUMERIC", "NULLABLE"), + schema.SchemaField("numeric_column", "NUMERIC", "NULLABLE"), ) assert returned_schema == expected_schema From dd833ba7624f5170c6c303a0b8cf64bb59e99ee5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Swe=C3=B1a=20=28Swast=29?= Date: Mon, 6 Jan 2025 10:54:42 -0600 Subject: [PATCH 456/519] chore: add bigframes team as co-owners for pandas-gbq (#847) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: add bigframes team as co-owners for pandas-gbq * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/CODEOWNERS | 8 ++++---- packages/pandas-gbq/.github/blunderbuss.yml | 3 +++ packages/pandas-gbq/.repo-metadata.json | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/pandas-gbq/.github/CODEOWNERS b/packages/pandas-gbq/.github/CODEOWNERS index 193b4363d07e..24c0ca968a01 100644 --- a/packages/pandas-gbq/.github/CODEOWNERS +++ b/packages/pandas-gbq/.github/CODEOWNERS @@ -5,8 +5,8 @@ # https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners#codeowners-syntax # Note: This file is autogenerated. To make changes to the codeowner team, please update .repo-metadata.json. -# @googleapis/yoshi-python @googleapis/api-bigquery are the default owners for changes in this repo -* @googleapis/yoshi-python @googleapis/api-bigquery +# @googleapis/yoshi-python @googleapis/api-bigquery @googleapis/api-bigquery-dataframe are the default owners for changes in this repo +* @googleapis/yoshi-python @googleapis/api-bigquery @googleapis/api-bigquery-dataframe -# @googleapis/python-samples-reviewers @googleapis/api-bigquery are the default owners for samples changes -/samples/ @googleapis/python-samples-reviewers @googleapis/api-bigquery +# @googleapis/python-samples-reviewers @googleapis/api-bigquery @googleapis/api-bigquery-dataframe are the default owners for samples changes +/samples/ @googleapis/python-samples-reviewers @googleapis/api-bigquery @googleapis/api-bigquery-dataframe diff --git a/packages/pandas-gbq/.github/blunderbuss.yml b/packages/pandas-gbq/.github/blunderbuss.yml index 5b7383dc7665..6677a65c995a 100644 --- a/packages/pandas-gbq/.github/blunderbuss.yml +++ b/packages/pandas-gbq/.github/blunderbuss.yml @@ -5,6 +5,7 @@ # team, please update `codeowner_team` in `.repo-metadata.json`. assign_issues: - googleapis/api-bigquery + - googleapis/api-bigquery-dataframe assign_issues_by: - labels: @@ -12,6 +13,8 @@ assign_issues_by: to: - googleapis/python-samples-reviewers - googleapis/api-bigquery + - googleapis/api-bigquery-dataframe assign_prs: - googleapis/api-bigquery + - googleapis/api-bigquery-dataframe diff --git a/packages/pandas-gbq/.repo-metadata.json b/packages/pandas-gbq/.repo-metadata.json index 912be418970a..b016c47d5e3f 100644 --- a/packages/pandas-gbq/.repo-metadata.json +++ b/packages/pandas-gbq/.repo-metadata.json @@ -11,5 +11,5 @@ "distribution_name": "pandas-gbq", "api_id": "bigquery.googleapis.com", "default_version": "", - "codeowner_team": "@googleapis/api-bigquery" + "codeowner_team": "@googleapis/api-bigquery @googleapis/api-bigquery-dataframe" } From a442dc34e435208beea9e0536c2af5a4c92d1f2d Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 6 Jan 2025 10:15:00 -0800 Subject: [PATCH 457/519] chore(main): release 0.26.1 (#848) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> Co-authored-by: Tim Sweña (Swast) --- packages/pandas-gbq/CHANGELOG.md | 7 +++++++ packages/pandas-gbq/pandas_gbq/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index 41b4c8df27d3..18fec1f2108e 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.26.1](https://github.com/googleapis/python-bigquery-pandas/compare/v0.26.0...v0.26.1) (2025-01-06) + + +### Bug Fixes + +* Ensure BIGNUMERIC type is used if scale > 9 in Decimal values ([#844](https://github.com/googleapis/python-bigquery-pandas/issues/844)) ([d2f32df](https://github.com/googleapis/python-bigquery-pandas/commit/d2f32df4670f4c18464c6772896bf1583c36e338)) + ## [0.26.0](https://github.com/googleapis/python-bigquery-pandas/compare/v0.25.0...v0.26.0) (2024-12-19) diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py index 0c8dab154f65..47551e36431c 100644 --- a/packages/pandas-gbq/pandas_gbq/version.py +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -2,4 +2,4 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -__version__ = "0.26.0" +__version__ = "0.26.1" From 04e2b9115318b542e5b46e4e30f6dc1353039a82 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Wed, 8 Jan 2025 17:25:22 +0100 Subject: [PATCH 458/519] chore(deps): update all dependencies (#831) --- .../pandas-gbq/.kokoro/docker/docs/requirements.txt | 12 ++++++------ .../samples/snippets/requirements-test.txt | 2 +- .../pandas-gbq/samples/snippets/requirements.txt | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt b/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt index f99a5c4aac7f..48ace5de9164 100644 --- a/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt +++ b/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt @@ -4,9 +4,9 @@ # # pip-compile --allow-unsafe --generate-hashes synthtool/gcp/templates/python_library/.kokoro/docker/docs/requirements.in # -argcomplete==3.5.2 \ - --hash=sha256:036d020d79048a5d525bc63880d7a4b8d1668566b8a76daf1144c0bbe0f63472 \ - --hash=sha256:23146ed7ac4403b70bd6026402468942ceba34a6732255b9edf5b7354f68a6bb +argcomplete==3.5.3 \ + --hash=sha256:2ab2c4a215c59fd6caaff41a869480a23e8f6a5f910b266c1808037f4e375b61 \ + --hash=sha256:c12bf50eded8aebb298c7b7da7a5ff3ee24dffd9f5281867dfe1424b58c55392 # via nox colorlog==6.9.0 \ --hash=sha256:5906e71acd67cb07a71e779c47c4bcb45fb8c2993eebe9e5adcd6a6f1b283eff \ @@ -66,7 +66,7 @@ tomli==2.2.1 \ --hash=sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a \ --hash=sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7 # via nox -virtualenv==20.28.0 \ - --hash=sha256:23eae1b4516ecd610481eda647f3a7c09aea295055337331bb4e6892ecce47b0 \ - --hash=sha256:2c9c3262bb8e7b87ea801d715fae4495e6032450c71d2309be9550e7364049aa +virtualenv==20.28.1 \ + --hash=sha256:412773c85d4dab0409b83ec36f7a6499e72eaf08c80e81e9576bca61831c71cb \ + --hash=sha256:5d34ab240fdb5d21549b76f9e8ff3af28252f5499fb6d6f031adac4e5a8c5329 # via nox diff --git a/packages/pandas-gbq/samples/snippets/requirements-test.txt b/packages/pandas-gbq/samples/snippets/requirements-test.txt index e1f01f29fe81..48d1a2e4fe74 100644 --- a/packages/pandas-gbq/samples/snippets/requirements-test.txt +++ b/packages/pandas-gbq/samples/snippets/requirements-test.txt @@ -1,2 +1,2 @@ -google-cloud-testutils==1.4.0 +google-cloud-testutils==1.5.0 pytest==8.3.4 diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 3ae34fd16835..2bfdd4a1ba43 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,6 +1,6 @@ google-cloud-bigquery-storage==2.27.0 google-cloud-bigquery==3.27.0 -pandas-gbq==0.24.0 +pandas-gbq==0.26.1 pandas===2.0.3; python_version == '3.8' pandas==2.2.3; python_version >= '3.9' pyarrow==18.1.0; python_version >= '3.9' From b6f57b138a93989d9e5621cd13b181459dadb479 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jan 2025 02:44:39 +0800 Subject: [PATCH 459/519] chore(deps): bump jinja2 from 3.1.4 to 3.1.5 in /.kokoro (#849) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): bump jinja2 from 3.1.4 to 3.1.5 in /.kokoro Bumps [jinja2](https://github.com/pallets/jinja) from 3.1.4 to 3.1.5. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/3.1.4...3.1.5) --- updated-dependencies: - dependency-name: jinja2 dependency-type: indirect ... Signed-off-by: dependabot[bot] * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Owl Bot --- .../pandas-gbq/.kokoro/docker/docs/requirements.txt | 12 ++++++------ packages/pandas-gbq/.kokoro/requirements.txt | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt b/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt index 48ace5de9164..f99a5c4aac7f 100644 --- a/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt +++ b/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt @@ -4,9 +4,9 @@ # # pip-compile --allow-unsafe --generate-hashes synthtool/gcp/templates/python_library/.kokoro/docker/docs/requirements.in # -argcomplete==3.5.3 \ - --hash=sha256:2ab2c4a215c59fd6caaff41a869480a23e8f6a5f910b266c1808037f4e375b61 \ - --hash=sha256:c12bf50eded8aebb298c7b7da7a5ff3ee24dffd9f5281867dfe1424b58c55392 +argcomplete==3.5.2 \ + --hash=sha256:036d020d79048a5d525bc63880d7a4b8d1668566b8a76daf1144c0bbe0f63472 \ + --hash=sha256:23146ed7ac4403b70bd6026402468942ceba34a6732255b9edf5b7354f68a6bb # via nox colorlog==6.9.0 \ --hash=sha256:5906e71acd67cb07a71e779c47c4bcb45fb8c2993eebe9e5adcd6a6f1b283eff \ @@ -66,7 +66,7 @@ tomli==2.2.1 \ --hash=sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a \ --hash=sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7 # via nox -virtualenv==20.28.1 \ - --hash=sha256:412773c85d4dab0409b83ec36f7a6499e72eaf08c80e81e9576bca61831c71cb \ - --hash=sha256:5d34ab240fdb5d21549b76f9e8ff3af28252f5499fb6d6f031adac4e5a8c5329 +virtualenv==20.28.0 \ + --hash=sha256:23eae1b4516ecd610481eda647f3a7c09aea295055337331bb4e6892ecce47b0 \ + --hash=sha256:2c9c3262bb8e7b87ea801d715fae4495e6032450c71d2309be9550e7364049aa # via nox diff --git a/packages/pandas-gbq/.kokoro/requirements.txt b/packages/pandas-gbq/.kokoro/requirements.txt index 006d8ef931bf..16db448c16bf 100644 --- a/packages/pandas-gbq/.kokoro/requirements.txt +++ b/packages/pandas-gbq/.kokoro/requirements.txt @@ -254,9 +254,9 @@ jeepney==0.8.0 \ # via # keyring # secretstorage -jinja2==3.1.4 \ - --hash=sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369 \ - --hash=sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d +jinja2==3.1.5 \ + --hash=sha256:8fefff8dc3034e27bb80d67c671eb8a9bc424c0ef4c0826edbff304cceff43bb \ + --hash=sha256:aba0f4dc9ed8013c424088f68a5c226f7d6097ed89b246d7749c2ec4175c6adb # via gcp-releasetool keyring==25.4.1 \ --hash=sha256:5426f817cf7f6f007ba5ec722b1bcad95a75b27d780343772ad76b17cb47b0bf \ From 854bf8955e13e261f4e2132bcf059f8292c5352b Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Thu, 9 Jan 2025 00:02:05 +0100 Subject: [PATCH 460/519] chore(deps): update all dependencies (#850) --- .../pandas-gbq/.kokoro/docker/docs/requirements.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt b/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt index f99a5c4aac7f..48ace5de9164 100644 --- a/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt +++ b/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt @@ -4,9 +4,9 @@ # # pip-compile --allow-unsafe --generate-hashes synthtool/gcp/templates/python_library/.kokoro/docker/docs/requirements.in # -argcomplete==3.5.2 \ - --hash=sha256:036d020d79048a5d525bc63880d7a4b8d1668566b8a76daf1144c0bbe0f63472 \ - --hash=sha256:23146ed7ac4403b70bd6026402468942ceba34a6732255b9edf5b7354f68a6bb +argcomplete==3.5.3 \ + --hash=sha256:2ab2c4a215c59fd6caaff41a869480a23e8f6a5f910b266c1808037f4e375b61 \ + --hash=sha256:c12bf50eded8aebb298c7b7da7a5ff3ee24dffd9f5281867dfe1424b58c55392 # via nox colorlog==6.9.0 \ --hash=sha256:5906e71acd67cb07a71e779c47c4bcb45fb8c2993eebe9e5adcd6a6f1b283eff \ @@ -66,7 +66,7 @@ tomli==2.2.1 \ --hash=sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a \ --hash=sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7 # via nox -virtualenv==20.28.0 \ - --hash=sha256:23eae1b4516ecd610481eda647f3a7c09aea295055337331bb4e6892ecce47b0 \ - --hash=sha256:2c9c3262bb8e7b87ea801d715fae4495e6032450c71d2309be9550e7364049aa +virtualenv==20.28.1 \ + --hash=sha256:412773c85d4dab0409b83ec36f7a6499e72eaf08c80e81e9576bca61831c71cb \ + --hash=sha256:5d34ab240fdb5d21549b76f9e8ff3af28252f5499fb6d6f031adac4e5a8c5329 # via nox From ff20a0ccd92849cce1d12c54a5a8af2c20c31137 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Mon, 13 Jan 2025 17:02:54 -0600 Subject: [PATCH 461/519] chore(python): exclude .github/workflows/unittest.yml in renovate config (#851) Source-Link: https://github.com/googleapis/synthtool/commit/106d292bd234e5d9977231dcfbc4831e34eba13a Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:8ff1efe878e18bd82a0fb7b70bb86f77e7ab6901fed394440b6135db0ba8d84a Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 6 +++--- packages/pandas-gbq/.github/workflows/unittest.yml | 5 ++++- .../pandas-gbq/.kokoro/docker/docs/requirements.txt | 12 ++++++------ packages/pandas-gbq/renovate.json | 2 +- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 26306af66f81..10cf433a8b00 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -1,4 +1,4 @@ -# Copyright 2024 Google LLC +# Copyright 2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:8e3e7e18255c22d1489258d0374c901c01f9c4fd77a12088670cd73d580aa737 -# created: 2024-12-17T00:59:58.625514486Z + digest: sha256:8ff1efe878e18bd82a0fb7b70bb86f77e7ab6901fed394440b6135db0ba8d84a +# created: 2025-01-09T12:01:16.422459506Z diff --git a/packages/pandas-gbq/.github/workflows/unittest.yml b/packages/pandas-gbq/.github/workflows/unittest.yml index 7be33c8f1b48..ece9c4c6c0cf 100644 --- a/packages/pandas-gbq/.github/workflows/unittest.yml +++ b/packages/pandas-gbq/.github/workflows/unittest.yml @@ -5,7 +5,10 @@ on: name: unittest jobs: unit: - runs-on: ubuntu-latest + # TODO(https://github.com/googleapis/gapic-generator-python/issues/2303): use `ubuntu-latest` once this bug is fixed. + # Use ubuntu-22.04 until Python 3.7 is removed from the test matrix + # https://docs.github.com/en/actions/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories + runs-on: ubuntu-22.04 strategy: matrix: python: ['3.8', '3.9', '3.10', '3.11', '3.12'] diff --git a/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt b/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt index 48ace5de9164..f99a5c4aac7f 100644 --- a/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt +++ b/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt @@ -4,9 +4,9 @@ # # pip-compile --allow-unsafe --generate-hashes synthtool/gcp/templates/python_library/.kokoro/docker/docs/requirements.in # -argcomplete==3.5.3 \ - --hash=sha256:2ab2c4a215c59fd6caaff41a869480a23e8f6a5f910b266c1808037f4e375b61 \ - --hash=sha256:c12bf50eded8aebb298c7b7da7a5ff3ee24dffd9f5281867dfe1424b58c55392 +argcomplete==3.5.2 \ + --hash=sha256:036d020d79048a5d525bc63880d7a4b8d1668566b8a76daf1144c0bbe0f63472 \ + --hash=sha256:23146ed7ac4403b70bd6026402468942ceba34a6732255b9edf5b7354f68a6bb # via nox colorlog==6.9.0 \ --hash=sha256:5906e71acd67cb07a71e779c47c4bcb45fb8c2993eebe9e5adcd6a6f1b283eff \ @@ -66,7 +66,7 @@ tomli==2.2.1 \ --hash=sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a \ --hash=sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7 # via nox -virtualenv==20.28.1 \ - --hash=sha256:412773c85d4dab0409b83ec36f7a6499e72eaf08c80e81e9576bca61831c71cb \ - --hash=sha256:5d34ab240fdb5d21549b76f9e8ff3af28252f5499fb6d6f031adac4e5a8c5329 +virtualenv==20.28.0 \ + --hash=sha256:23eae1b4516ecd610481eda647f3a7c09aea295055337331bb4e6892ecce47b0 \ + --hash=sha256:2c9c3262bb8e7b87ea801d715fae4495e6032450c71d2309be9550e7364049aa # via nox diff --git a/packages/pandas-gbq/renovate.json b/packages/pandas-gbq/renovate.json index 39b2a0ec9296..c7875c469bd5 100644 --- a/packages/pandas-gbq/renovate.json +++ b/packages/pandas-gbq/renovate.json @@ -5,7 +5,7 @@ ":preserveSemverRanges", ":disableDependencyDashboard" ], - "ignorePaths": [".pre-commit-config.yaml", ".kokoro/requirements.txt", "setup.py"], + "ignorePaths": [".pre-commit-config.yaml", ".kokoro/requirements.txt", "setup.py", ".github/workflows/unittest.yml"], "pip_requirements": { "fileMatch": ["requirements-test.txt", "samples/[\\S/]*constraints.txt", "samples/[\\S/]*constraints-test.txt"] } From e0438ac77a2bb0a3a0eedf6e523e5fdff4cd10c1 Mon Sep 17 00:00:00 2001 From: Lingqing Gan Date: Tue, 21 Jan 2025 09:55:13 -0800 Subject: [PATCH 462/519] test: use main branch of python-bigquery and python-bigquery-storage for pre-release tests (#857) --- packages/pandas-gbq/noxfile.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 461b761c6a7d..cf3405af6210 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -324,8 +324,6 @@ def prerelease(session): "--pre", "--upgrade", "google-api-core", - "google-cloud-bigquery", - "google-cloud-bigquery-storage", "google-cloud-core", "google-resumable-media", # Exclude version 1.49.0rc1 which has a known issue. See https://github.com/grpc/grpc/pull/30642 @@ -344,6 +342,14 @@ def prerelease(session): "pytest-cov", ) + # Install python-bigquery and python-bigquery-storage from main to detect + # any potential breaking changes. For context, see: + # https://github.com/googleapis/python-bigquery-pandas/issues/854 + session.install( + "https://github.com/googleapis/python-bigquery/archive/main.zip", + "https://github.com/googleapis/python-bigquery-storage/archive/main.zip", + ) + # Because we test minimum dependency versions on the minimum Python # version, the first version we test with in the unit tests sessions has a # constraints file containing all dependencies and extras. From b743ef1d4087e80ebee9fadcf710c4fd356ba531 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Wed, 5 Feb 2025 13:26:29 -0800 Subject: [PATCH 463/519] chore(python): fix docs publish build (#856) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Source-Link: https://github.com/googleapis/synthtool/commit/bd9ede2fea1b640b7e90d5a1d110e6b300a2b43f Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:04c35dc5f49f0f503a306397d6d043685f8d2bb822ab515818c4208d7fb2db3a Co-authored-by: Owl Bot Co-authored-by: Tim Sweña (Swast) --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 +- .../.kokoro/docker/docs/requirements.in | 1 + .../.kokoro/docker/docs/requirements.txt | 243 +++++++++++++++++- packages/pandas-gbq/.kokoro/publish-docs.sh | 4 - 4 files changed, 237 insertions(+), 15 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 10cf433a8b00..4c0027ff1c61 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:8ff1efe878e18bd82a0fb7b70bb86f77e7ab6901fed394440b6135db0ba8d84a -# created: 2025-01-09T12:01:16.422459506Z + digest: sha256:04c35dc5f49f0f503a306397d6d043685f8d2bb822ab515818c4208d7fb2db3a +# created: 2025-01-16T15:24:11.364245182Z diff --git a/packages/pandas-gbq/.kokoro/docker/docs/requirements.in b/packages/pandas-gbq/.kokoro/docker/docs/requirements.in index 816817c672a1..586bd07037ae 100644 --- a/packages/pandas-gbq/.kokoro/docker/docs/requirements.in +++ b/packages/pandas-gbq/.kokoro/docker/docs/requirements.in @@ -1 +1,2 @@ nox +gcp-docuploader diff --git a/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt b/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt index f99a5c4aac7f..a9360a25b707 100644 --- a/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt +++ b/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt @@ -2,16 +2,124 @@ # This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile --allow-unsafe --generate-hashes synthtool/gcp/templates/python_library/.kokoro/docker/docs/requirements.in +# pip-compile --allow-unsafe --generate-hashes requirements.in # -argcomplete==3.5.2 \ - --hash=sha256:036d020d79048a5d525bc63880d7a4b8d1668566b8a76daf1144c0bbe0f63472 \ - --hash=sha256:23146ed7ac4403b70bd6026402468942ceba34a6732255b9edf5b7354f68a6bb +argcomplete==3.5.3 \ + --hash=sha256:2ab2c4a215c59fd6caaff41a869480a23e8f6a5f910b266c1808037f4e375b61 \ + --hash=sha256:c12bf50eded8aebb298c7b7da7a5ff3ee24dffd9f5281867dfe1424b58c55392 # via nox +cachetools==5.5.0 \ + --hash=sha256:02134e8439cdc2ffb62023ce1debca2944c3f289d66bb17ead3ab3dede74b292 \ + --hash=sha256:2cc24fb4cbe39633fb7badd9db9ca6295d766d9c2995f245725a46715d050f2a + # via google-auth +certifi==2024.12.14 \ + --hash=sha256:1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56 \ + --hash=sha256:b650d30f370c2b724812bee08008be0c4163b163ddaec3f2546c1caf65f191db + # via requests +charset-normalizer==3.4.1 \ + --hash=sha256:0167ddc8ab6508fe81860a57dd472b2ef4060e8d378f0cc555707126830f2537 \ + --hash=sha256:01732659ba9b5b873fc117534143e4feefecf3b2078b0a6a2e925271bb6f4cfa \ + --hash=sha256:01ad647cdd609225c5350561d084b42ddf732f4eeefe6e678765636791e78b9a \ + --hash=sha256:04432ad9479fa40ec0f387795ddad4437a2b50417c69fa275e212933519ff294 \ + --hash=sha256:0907f11d019260cdc3f94fbdb23ff9125f6b5d1039b76003b5b0ac9d6a6c9d5b \ + --hash=sha256:0924e81d3d5e70f8126529951dac65c1010cdf117bb75eb02dd12339b57749dd \ + --hash=sha256:09b26ae6b1abf0d27570633b2b078a2a20419c99d66fb2823173d73f188ce601 \ + --hash=sha256:09b5e6733cbd160dcc09589227187e242a30a49ca5cefa5a7edd3f9d19ed53fd \ + --hash=sha256:0af291f4fe114be0280cdd29d533696a77b5b49cfde5467176ecab32353395c4 \ + --hash=sha256:0f55e69f030f7163dffe9fd0752b32f070566451afe180f99dbeeb81f511ad8d \ + --hash=sha256:1a2bc9f351a75ef49d664206d51f8e5ede9da246602dc2d2726837620ea034b2 \ + --hash=sha256:22e14b5d70560b8dd51ec22863f370d1e595ac3d024cb8ad7d308b4cd95f8313 \ + --hash=sha256:234ac59ea147c59ee4da87a0c0f098e9c8d169f4dc2a159ef720f1a61bbe27cd \ + --hash=sha256:2369eea1ee4a7610a860d88f268eb39b95cb588acd7235e02fd5a5601773d4fa \ + --hash=sha256:237bdbe6159cff53b4f24f397d43c6336c6b0b42affbe857970cefbb620911c8 \ + --hash=sha256:28bf57629c75e810b6ae989f03c0828d64d6b26a5e205535585f96093e405ed1 \ + --hash=sha256:2967f74ad52c3b98de4c3b32e1a44e32975e008a9cd2a8cc8966d6a5218c5cb2 \ + --hash=sha256:2a75d49014d118e4198bcee5ee0a6f25856b29b12dbf7cd012791f8a6cc5c496 \ + --hash=sha256:2bdfe3ac2e1bbe5b59a1a63721eb3b95fc9b6817ae4a46debbb4e11f6232428d \ + --hash=sha256:2d074908e1aecee37a7635990b2c6d504cd4766c7bc9fc86d63f9c09af3fa11b \ + --hash=sha256:2fb9bd477fdea8684f78791a6de97a953c51831ee2981f8e4f583ff3b9d9687e \ + --hash=sha256:311f30128d7d333eebd7896965bfcfbd0065f1716ec92bd5638d7748eb6f936a \ + --hash=sha256:329ce159e82018d646c7ac45b01a430369d526569ec08516081727a20e9e4af4 \ + --hash=sha256:345b0426edd4e18138d6528aed636de7a9ed169b4aaf9d61a8c19e39d26838ca \ + --hash=sha256:363e2f92b0f0174b2f8238240a1a30142e3db7b957a5dd5689b0e75fb717cc78 \ + --hash=sha256:3a3bd0dcd373514dcec91c411ddb9632c0d7d92aed7093b8c3bbb6d69ca74408 \ + --hash=sha256:3bed14e9c89dcb10e8f3a29f9ccac4955aebe93c71ae803af79265c9ca5644c5 \ + --hash=sha256:44251f18cd68a75b56585dd00dae26183e102cd5e0f9f1466e6df5da2ed64ea3 \ + --hash=sha256:44ecbf16649486d4aebafeaa7ec4c9fed8b88101f4dd612dcaf65d5e815f837f \ + --hash=sha256:4532bff1b8421fd0a320463030c7520f56a79c9024a4e88f01c537316019005a \ + --hash=sha256:49402233c892a461407c512a19435d1ce275543138294f7ef013f0b63d5d3765 \ + --hash=sha256:4c0907b1928a36d5a998d72d64d8eaa7244989f7aaaf947500d3a800c83a3fd6 \ + --hash=sha256:4d86f7aff21ee58f26dcf5ae81a9addbd914115cdebcbb2217e4f0ed8982e146 \ + --hash=sha256:5777ee0881f9499ed0f71cc82cf873d9a0ca8af166dfa0af8ec4e675b7df48e6 \ + --hash=sha256:5df196eb874dae23dcfb968c83d4f8fdccb333330fe1fc278ac5ceeb101003a9 \ + --hash=sha256:619a609aa74ae43d90ed2e89bdd784765de0a25ca761b93e196d938b8fd1dbbd \ + --hash=sha256:6e27f48bcd0957c6d4cb9d6fa6b61d192d0b13d5ef563e5f2ae35feafc0d179c \ + --hash=sha256:6ff8a4a60c227ad87030d76e99cd1698345d4491638dfa6673027c48b3cd395f \ + --hash=sha256:73d94b58ec7fecbc7366247d3b0b10a21681004153238750bb67bd9012414545 \ + --hash=sha256:7461baadb4dc00fd9e0acbe254e3d7d2112e7f92ced2adc96e54ef6501c5f176 \ + --hash=sha256:75832c08354f595c760a804588b9357d34ec00ba1c940c15e31e96d902093770 \ + --hash=sha256:7709f51f5f7c853f0fb938bcd3bc59cdfdc5203635ffd18bf354f6967ea0f824 \ + --hash=sha256:78baa6d91634dfb69ec52a463534bc0df05dbd546209b79a3880a34487f4b84f \ + --hash=sha256:7974a0b5ecd505609e3b19742b60cee7aa2aa2fb3151bc917e6e2646d7667dcf \ + --hash=sha256:7a4f97a081603d2050bfaffdefa5b02a9ec823f8348a572e39032caa8404a487 \ + --hash=sha256:7b1bef6280950ee6c177b326508f86cad7ad4dff12454483b51d8b7d673a2c5d \ + --hash=sha256:7d053096f67cd1241601111b698f5cad775f97ab25d81567d3f59219b5f1adbd \ + --hash=sha256:804a4d582ba6e5b747c625bf1255e6b1507465494a40a2130978bda7b932c90b \ + --hash=sha256:807f52c1f798eef6cf26beb819eeb8819b1622ddfeef9d0977a8502d4db6d534 \ + --hash=sha256:80ed5e856eb7f30115aaf94e4a08114ccc8813e6ed1b5efa74f9f82e8509858f \ + --hash=sha256:8417cb1f36cc0bc7eaba8ccb0e04d55f0ee52df06df3ad55259b9a323555fc8b \ + --hash=sha256:8436c508b408b82d87dc5f62496973a1805cd46727c34440b0d29d8a2f50a6c9 \ + --hash=sha256:89149166622f4db9b4b6a449256291dc87a99ee53151c74cbd82a53c8c2f6ccd \ + --hash=sha256:8bfa33f4f2672964266e940dd22a195989ba31669bd84629f05fab3ef4e2d125 \ + --hash=sha256:8c60ca7339acd497a55b0ea5d506b2a2612afb2826560416f6894e8b5770d4a9 \ + --hash=sha256:91b36a978b5ae0ee86c394f5a54d6ef44db1de0815eb43de826d41d21e4af3de \ + --hash=sha256:955f8851919303c92343d2f66165294848d57e9bba6cf6e3625485a70a038d11 \ + --hash=sha256:97f68b8d6831127e4787ad15e6757232e14e12060bec17091b85eb1486b91d8d \ + --hash=sha256:9b23ca7ef998bc739bf6ffc077c2116917eabcc901f88da1b9856b210ef63f35 \ + --hash=sha256:9f0b8b1c6d84c8034a44893aba5e767bf9c7a211e313a9605d9c617d7083829f \ + --hash=sha256:aabfa34badd18f1da5ec1bc2715cadc8dca465868a4e73a0173466b688f29dda \ + --hash=sha256:ab36c8eb7e454e34e60eb55ca5d241a5d18b2c6244f6827a30e451c42410b5f7 \ + --hash=sha256:b010a7a4fd316c3c484d482922d13044979e78d1861f0e0650423144c616a46a \ + --hash=sha256:b1ac5992a838106edb89654e0aebfc24f5848ae2547d22c2c3f66454daa11971 \ + --hash=sha256:b7b2d86dd06bfc2ade3312a83a5c364c7ec2e3498f8734282c6c3d4b07b346b8 \ + --hash=sha256:b97e690a2118911e39b4042088092771b4ae3fc3aa86518f84b8cf6888dbdb41 \ + --hash=sha256:bc2722592d8998c870fa4e290c2eec2c1569b87fe58618e67d38b4665dfa680d \ + --hash=sha256:c0429126cf75e16c4f0ad00ee0eae4242dc652290f940152ca8c75c3a4b6ee8f \ + --hash=sha256:c30197aa96e8eed02200a83fba2657b4c3acd0f0aa4bdc9f6c1af8e8962e0757 \ + --hash=sha256:c4c3e6da02df6fa1410a7680bd3f63d4f710232d3139089536310d027950696a \ + --hash=sha256:c75cb2a3e389853835e84a2d8fb2b81a10645b503eca9bcb98df6b5a43eb8886 \ + --hash=sha256:c96836c97b1238e9c9e3fe90844c947d5afbf4f4c92762679acfe19927d81d77 \ + --hash=sha256:d7f50a1f8c450f3925cb367d011448c39239bb3eb4117c36a6d354794de4ce76 \ + --hash=sha256:d973f03c0cb71c5ed99037b870f2be986c3c05e63622c017ea9816881d2dd247 \ + --hash=sha256:d98b1668f06378c6dbefec3b92299716b931cd4e6061f3c875a71ced1780ab85 \ + --hash=sha256:d9c3cdf5390dcd29aa8056d13e8e99526cda0305acc038b96b30352aff5ff2bb \ + --hash=sha256:dad3e487649f498dd991eeb901125411559b22e8d7ab25d3aeb1af367df5efd7 \ + --hash=sha256:dccbe65bd2f7f7ec22c4ff99ed56faa1e9f785482b9bbd7c717e26fd723a1d1e \ + --hash=sha256:dd78cfcda14a1ef52584dbb008f7ac81c1328c0f58184bf9a84c49c605002da6 \ + --hash=sha256:e218488cd232553829be0664c2292d3af2eeeb94b32bea483cf79ac6a694e037 \ + --hash=sha256:e358e64305fe12299a08e08978f51fc21fac060dcfcddd95453eabe5b93ed0e1 \ + --hash=sha256:ea0d8d539afa5eb2728aa1932a988a9a7af94f18582ffae4bc10b3fbdad0626e \ + --hash=sha256:eab677309cdb30d047996b36d34caeda1dc91149e4fdca0b1a039b3f79d9a807 \ + --hash=sha256:eb8178fe3dba6450a3e024e95ac49ed3400e506fd4e9e5c32d30adda88cbd407 \ + --hash=sha256:ecddf25bee22fe4fe3737a399d0d177d72bc22be6913acfab364b40bce1ba83c \ + --hash=sha256:eea6ee1db730b3483adf394ea72f808b6e18cf3cb6454b4d86e04fa8c4327a12 \ + --hash=sha256:f08ff5e948271dc7e18a35641d2f11a4cd8dfd5634f55228b691e62b37125eb3 \ + --hash=sha256:f30bf9fd9be89ecb2360c7d94a711f00c09b976258846efe40db3d05828e8089 \ + --hash=sha256:fa88b843d6e211393a37219e6a1c1df99d35e8fd90446f1118f4216e307e48cd \ + --hash=sha256:fc54db6c8593ef7d4b2a331b58653356cf04f67c960f584edb7c3d8c97e8f39e \ + --hash=sha256:fd4ec41f914fa74ad1b8304bbc634b3de73d2a0889bd32076342a573e0779e00 \ + --hash=sha256:ffc9202a29ab3920fa812879e95a9e78b2465fd10be7fcbd042899695d75e616 + # via requests +click==8.1.8 \ + --hash=sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2 \ + --hash=sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a + # via gcp-docuploader colorlog==6.9.0 \ --hash=sha256:5906e71acd67cb07a71e779c47c4bcb45fb8c2993eebe9e5adcd6a6f1b283eff \ --hash=sha256:bfba54a1b93b94f54e1f4fe48395725a3d92fd2a4af702f6bd70946bdc0c6ac2 - # via nox + # via + # gcp-docuploader + # nox distlib==0.3.9 \ --hash=sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87 \ --hash=sha256:a60f20dea646b8a33f3e7772f74dc0b2d0772d2837ee1342a00645c81edf9403 @@ -20,10 +128,78 @@ filelock==3.16.1 \ --hash=sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0 \ --hash=sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435 # via virtualenv +gcp-docuploader==0.6.5 \ + --hash=sha256:30221d4ac3e5a2b9c69aa52fdbef68cc3f27d0e6d0d90e220fc024584b8d2318 \ + --hash=sha256:b7458ef93f605b9d46a4bf3a8dc1755dad1f31d030c8679edf304e343b347eea + # via -r requirements.in +google-api-core==2.24.0 \ + --hash=sha256:10d82ac0fca69c82a25b3efdeefccf6f28e02ebb97925a8cce8edbfe379929d9 \ + --hash=sha256:e255640547a597a4da010876d333208ddac417d60add22b6851a0c66a831fcaf + # via + # google-cloud-core + # google-cloud-storage +google-auth==2.37.0 \ + --hash=sha256:0054623abf1f9c83492c63d3f47e77f0a544caa3d40b2d98e099a611c2dd5d00 \ + --hash=sha256:42664f18290a6be591be5329a96fe30184be1a1badb7292a7f686a9659de9ca0 + # via + # google-api-core + # google-cloud-core + # google-cloud-storage +google-cloud-core==2.4.1 \ + --hash=sha256:9b7749272a812bde58fff28868d0c5e2f585b82f37e09a1f6ed2d4d10f134073 \ + --hash=sha256:a9e6a4422b9ac5c29f79a0ede9485473338e2ce78d91f2370c01e730eab22e61 + # via google-cloud-storage +google-cloud-storage==2.19.0 \ + --hash=sha256:aeb971b5c29cf8ab98445082cbfe7b161a1f48ed275822f59ed3f1524ea54fba \ + --hash=sha256:cd05e9e7191ba6cb68934d8eb76054d9be4562aa89dbc4236feee4d7d51342b2 + # via gcp-docuploader +google-crc32c==1.6.0 \ + --hash=sha256:05e2d8c9a2f853ff116db9706b4a27350587f341eda835f46db3c0a8c8ce2f24 \ + --hash=sha256:18e311c64008f1f1379158158bb3f0c8d72635b9eb4f9545f8cf990c5668e59d \ + --hash=sha256:236c87a46cdf06384f614e9092b82c05f81bd34b80248021f729396a78e55d7e \ + --hash=sha256:35834855408429cecf495cac67ccbab802de269e948e27478b1e47dfb6465e57 \ + --hash=sha256:386122eeaaa76951a8196310432c5b0ef3b53590ef4c317ec7588ec554fec5d2 \ + --hash=sha256:40b05ab32a5067525670880eb5d169529089a26fe35dce8891127aeddc1950e8 \ + --hash=sha256:48abd62ca76a2cbe034542ed1b6aee851b6f28aaca4e6551b5599b6f3ef175cc \ + --hash=sha256:50cf2a96da226dcbff8671233ecf37bf6e95de98b2a2ebadbfdf455e6d05df42 \ + --hash=sha256:51c4f54dd8c6dfeb58d1df5e4f7f97df8abf17a36626a217f169893d1d7f3e9f \ + --hash=sha256:5bcc90b34df28a4b38653c36bb5ada35671ad105c99cfe915fb5bed7ad6924aa \ + --hash=sha256:62f6d4a29fea082ac4a3c9be5e415218255cf11684ac6ef5488eea0c9132689b \ + --hash=sha256:6eceb6ad197656a1ff49ebfbbfa870678c75be4344feb35ac1edf694309413dc \ + --hash=sha256:7aec8e88a3583515f9e0957fe4f5f6d8d4997e36d0f61624e70469771584c760 \ + --hash=sha256:91ca8145b060679ec9176e6de4f89b07363d6805bd4760631ef254905503598d \ + --hash=sha256:a184243544811e4a50d345838a883733461e67578959ac59964e43cca2c791e7 \ + --hash=sha256:a9e4b426c3702f3cd23b933436487eb34e01e00327fac20c9aebb68ccf34117d \ + --hash=sha256:bb0966e1c50d0ef5bc743312cc730b533491d60585a9a08f897274e57c3f70e0 \ + --hash=sha256:bb8b3c75bd157010459b15222c3fd30577042a7060e29d42dabce449c087f2b3 \ + --hash=sha256:bd5e7d2445d1a958c266bfa5d04c39932dc54093fa391736dbfdb0f1929c1fb3 \ + --hash=sha256:c87d98c7c4a69066fd31701c4e10d178a648c2cac3452e62c6b24dc51f9fcc00 \ + --hash=sha256:d2952396dc604544ea7476b33fe87faedc24d666fb0c2d5ac971a2b9576ab871 \ + --hash=sha256:d8797406499f28b5ef791f339594b0b5fdedf54e203b5066675c406ba69d705c \ + --hash=sha256:d9e9913f7bd69e093b81da4535ce27af842e7bf371cde42d1ae9e9bd382dc0e9 \ + --hash=sha256:e2806553238cd076f0a55bddab37a532b53580e699ed8e5606d0de1f856b5205 \ + --hash=sha256:ebab974b1687509e5c973b5c4b8b146683e101e102e17a86bd196ecaa4d099fc \ + --hash=sha256:ed767bf4ba90104c1216b68111613f0d5926fb3780660ea1198fc469af410e9d \ + --hash=sha256:f7a1fc29803712f80879b0806cb83ab24ce62fc8daf0569f2204a0cfd7f68ed4 + # via + # google-cloud-storage + # google-resumable-media +google-resumable-media==2.7.2 \ + --hash=sha256:3ce7551e9fe6d99e9a126101d2536612bb73486721951e9562fee0f90c6ababa \ + --hash=sha256:5280aed4629f2b60b847b0d42f9857fd4935c11af266744df33d8074cae92fe0 + # via google-cloud-storage +googleapis-common-protos==1.66.0 \ + --hash=sha256:c3e7b33d15fdca5374cc0a7346dd92ffa847425cc4ea941d970f13680052ec8c \ + --hash=sha256:d7abcd75fabb2e0ec9f74466401f6c119a0b498e27370e9be4c94cb7e382b8ed + # via google-api-core +idna==3.10 \ + --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \ + --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3 + # via requests nox==2024.10.9 \ --hash=sha256:1d36f309a0a2a853e9bccb76bbef6bb118ba92fa92674d15604ca99adeb29eab \ --hash=sha256:7aa9dc8d1c27e9f45ab046ffd1c3b2c4f7c91755304769df231308849ebded95 - # via -r synthtool/gcp/templates/python_library/.kokoro/docker/docs/requirements.in + # via -r requirements.in packaging==24.2 \ --hash=sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759 \ --hash=sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f @@ -32,6 +208,51 @@ platformdirs==4.3.6 \ --hash=sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907 \ --hash=sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb # via virtualenv +proto-plus==1.25.0 \ + --hash=sha256:c91fc4a65074ade8e458e95ef8bac34d4008daa7cce4a12d6707066fca648961 \ + --hash=sha256:fbb17f57f7bd05a68b7707e745e26528b0b3c34e378db91eef93912c54982d91 + # via google-api-core +protobuf==5.29.3 \ + --hash=sha256:0a18ed4a24198528f2333802eb075e59dea9d679ab7a6c5efb017a59004d849f \ + --hash=sha256:0eb32bfa5219fc8d4111803e9a690658aa2e6366384fd0851064b963b6d1f2a7 \ + --hash=sha256:3ea51771449e1035f26069c4c7fd51fba990d07bc55ba80701c78f886bf9c888 \ + --hash=sha256:5da0f41edaf117bde316404bad1a486cb4ededf8e4a54891296f648e8e076620 \ + --hash=sha256:6ce8cc3389a20693bfde6c6562e03474c40851b44975c9b2bf6df7d8c4f864da \ + --hash=sha256:84a57163a0ccef3f96e4b6a20516cedcf5bb3a95a657131c5c3ac62200d23252 \ + --hash=sha256:a4fa6f80816a9a0678429e84973f2f98cbc218cca434abe8db2ad0bffc98503a \ + --hash=sha256:a8434404bbf139aa9e1300dbf989667a83d42ddda9153d8ab76e0d5dcaca484e \ + --hash=sha256:b89c115d877892a512f79a8114564fb435943b59067615894c3b13cd3e1fa107 \ + --hash=sha256:c027e08a08be10b67c06bf2370b99c811c466398c357e615ca88c91c07f0910f \ + --hash=sha256:daaf63f70f25e8689c072cfad4334ca0ac1d1e05a92fc15c54eb9cf23c3efd84 + # via + # gcp-docuploader + # google-api-core + # googleapis-common-protos + # proto-plus +pyasn1==0.6.1 \ + --hash=sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629 \ + --hash=sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.4.1 \ + --hash=sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd \ + --hash=sha256:c28e2dbf9c06ad61c71a075c7e0f9fd0f1b0bb2d2ad4377f240d33ac2ab60a7c + # via google-auth +requests==2.32.3 \ + --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \ + --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 + # via + # google-api-core + # google-cloud-storage +rsa==4.9 \ + --hash=sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7 \ + --hash=sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21 + # via google-auth +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ + --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 + # via gcp-docuploader tomli==2.2.1 \ --hash=sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6 \ --hash=sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd \ @@ -66,7 +287,11 @@ tomli==2.2.1 \ --hash=sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a \ --hash=sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7 # via nox -virtualenv==20.28.0 \ - --hash=sha256:23eae1b4516ecd610481eda647f3a7c09aea295055337331bb4e6892ecce47b0 \ - --hash=sha256:2c9c3262bb8e7b87ea801d715fae4495e6032450c71d2309be9550e7364049aa +urllib3==2.3.0 \ + --hash=sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df \ + --hash=sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d + # via requests +virtualenv==20.28.1 \ + --hash=sha256:412773c85d4dab0409b83ec36f7a6499e72eaf08c80e81e9576bca61831c71cb \ + --hash=sha256:5d34ab240fdb5d21549b76f9e8ff3af28252f5499fb6d6f031adac4e5a8c5329 # via nox diff --git a/packages/pandas-gbq/.kokoro/publish-docs.sh b/packages/pandas-gbq/.kokoro/publish-docs.sh index 233205d580e9..4ed4aaf1346f 100755 --- a/packages/pandas-gbq/.kokoro/publish-docs.sh +++ b/packages/pandas-gbq/.kokoro/publish-docs.sh @@ -20,10 +20,6 @@ export PYTHONUNBUFFERED=1 export PATH="${HOME}/.local/bin:${PATH}" -# Install nox -python3.10 -m pip install --require-hashes -r .kokoro/requirements.txt -python3.10 -m nox --version - # build docs nox -s docs From 0f1048cc2e46471099e88af9523eb7c2c831f214 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Swe=C3=B1a=20=28Swast=29?= Date: Wed, 5 Feb 2025 16:33:13 -0600 Subject: [PATCH 464/519] feat: `to_gbq` can write non-string values to existing STRING columns in BigQuery (#876) --- .../pandas-gbq/pandas_gbq/load/__init__.py | 23 +++ .../pandas_gbq/{load.py => load/core.py} | 5 + .../pandas-gbq/tests/system/test_to_gbq.py | 155 ++++++++++++++---- 3 files changed, 148 insertions(+), 35 deletions(-) create mode 100644 packages/pandas-gbq/pandas_gbq/load/__init__.py rename packages/pandas-gbq/pandas_gbq/{load.py => load/core.py} (96%) diff --git a/packages/pandas-gbq/pandas_gbq/load/__init__.py b/packages/pandas-gbq/pandas_gbq/load/__init__.py new file mode 100644 index 000000000000..250d65176865 --- /dev/null +++ b/packages/pandas-gbq/pandas_gbq/load/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) 2025 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +from pandas_gbq.load.core import ( + cast_dataframe_for_parquet, + encode_chunk, + load_chunks, + load_csv_from_dataframe, + load_csv_from_file, + load_parquet, + split_dataframe, +) + +__all__ = [ + "cast_dataframe_for_parquet", + "encode_chunk", + "load_chunks", + "load_csv_from_dataframe", + "load_csv_from_file", + "load_parquet", + "split_dataframe", +] diff --git a/packages/pandas-gbq/pandas_gbq/load.py b/packages/pandas-gbq/pandas_gbq/load/core.py similarity index 96% rename from packages/pandas-gbq/pandas_gbq/load.py rename to packages/pandas-gbq/pandas_gbq/load/core.py index 567899df664a..d98f83068b7f 100644 --- a/packages/pandas-gbq/pandas_gbq/load.py +++ b/packages/pandas-gbq/pandas_gbq/load/core.py @@ -111,6 +111,11 @@ def convert(x): return decimal.Decimal(x) cast_column = dataframe[column_name].map(convert) + elif column_type == "STRING": + # Allow non-string columns to be uploaded to STRING in BigQuery. + # https://github.com/googleapis/python-bigquery-pandas/issues/875 + # TODO: Use pyarrow as the storage when the minimum pandas version allows for it. + cast_column = dataframe[column_name].astype(pandas.StringDtype()) else: cast_column = None diff --git a/packages/pandas-gbq/tests/system/test_to_gbq.py b/packages/pandas-gbq/tests/system/test_to_gbq.py index 6352fbd7620d..139f072b6abb 100644 --- a/packages/pandas-gbq/tests/system/test_to_gbq.py +++ b/packages/pandas-gbq/tests/system/test_to_gbq.py @@ -28,13 +28,13 @@ def method_under_test(to_gbq): SeriesRoundTripTestCase = collections.namedtuple( "SeriesRoundTripTestCase", - ["input_series", "api_methods"], - defaults=[None, {"load_csv", "load_parquet"}], + ["input_series", "api_methods", "expected_dtype"], + defaults=[None, {"load_csv", "load_parquet"}, None], ) @pytest.mark.parametrize( - ["input_series", "api_methods"], + ["input_series", "api_methods", "expected_dtype"], [ # Ensure that 64-bit floating point numbers are unchanged. # See: https://github.com/pydata/pandas-gbq/issues/326 @@ -53,40 +53,46 @@ def method_under_test(to_gbq): name="test_col", ), ), - SeriesRoundTripTestCase( - input_series=pandas.Series( - [ - "abc", - "defg", - # Ensure that unicode characters are encoded. See: - # https://github.com/googleapis/python-bigquery-pandas/issues/106 - "信用卡", - "Skywalker™", - "hülle", - ], - name="test_col", + pytest.param( + *SeriesRoundTripTestCase( + input_series=pandas.Series( + [ + "abc", + "defg", + # Ensure that unicode characters are encoded. See: + # https://github.com/googleapis/python-bigquery-pandas/issues/106 + "信用卡", + "Skywalker™", + "hülle", + ], + name="test_col", + ), ), + id="string-unicode", ), - SeriesRoundTripTestCase( - input_series=pandas.Series( - [ - "abc", - "defg", - # Ensure that empty strings are written as empty string, - # not NULL. See: - # https://github.com/googleapis/python-bigquery-pandas/issues/366 - "", - None, - ], - name="empty_strings", + pytest.param( + *SeriesRoundTripTestCase( + input_series=pandas.Series( + [ + "abc", + "defg", + # Ensure that empty strings are written as empty string, + # not NULL. See: + # https://github.com/googleapis/python-bigquery-pandas/issues/366 + "", + None, + ], + name="empty_strings", + ), + # BigQuery CSV loader uses empty string as the "null marker" by + # default. Potentially one could choose a rarely used character or + # string as the null marker to disambiguate null from empty string, + # but then that string couldn't be loaded. + # TODO: Revist when custom load job configuration is supported. + # https://github.com/googleapis/python-bigquery-pandas/issues/425 + api_methods={"load_parquet"}, ), - # BigQuery CSV loader uses empty string as the "null marker" by - # default. Potentially one could choose a rarely used character or - # string as the null marker to disambiguate null from empty string, - # but then that string couldn't be loaded. - # TODO: Revist when custom load job configuration is supported. - # https://github.com/googleapis/python-bigquery-pandas/issues/425 - api_methods={"load_parquet"}, + id="string-empty-and-null", ), ], ) @@ -97,6 +103,7 @@ def test_series_round_trip( input_series, api_method, api_methods, + expected_dtype, ): if api_method not in api_methods: pytest.skip(f"{api_method} not supported.") @@ -111,9 +118,14 @@ def test_series_round_trip( round_trip = read_gbq(table_id) round_trip_series = round_trip["test_col"].sort_values().reset_index(drop=True) + + expected_series = input_series.copy() + if expected_dtype is not None: + expected_series = expected_series.astype(expected_dtype) + pandas.testing.assert_series_equal( round_trip_series, - input_series, + expected_series, check_exact=True, check_names=False, ) @@ -362,6 +374,79 @@ def test_series_round_trip( ), id="issue365-extreme-datetimes", ), + # Loading a STRING column should work with all available string dtypes. + pytest.param( + *DataFrameRoundTripTestCase( + input_df=pandas.DataFrame( + { + "row_num": [1, 2, 3], + # If a cast to STRING is lossless, pandas-gbq should do that automatically. + # See: https://github.com/googleapis/python-bigquery-pandas/issues/875 + "int_want_string": [94043, 10011, 98033], + "object": pandas.Series(["a", "b", "c"], dtype="object"), + "string_python": pandas.Series( + ["d", "e", "f"], + dtype=( + pandas.StringDtype(storage="python") + if hasattr(pandas, "ArrowDtype") + else pandas.StringDtype() + ), + ), + "string_pyarrow": pandas.Series( + ["g", "h", "i"], + dtype=( + pandas.StringDtype(storage="pyarrow") + if hasattr(pandas, "ArrowDtype") + else pandas.StringDtype() + ), + ), + "arrowdtype_string": pandas.Series( + ["j", "k", "l"], + dtype=( + pandas.ArrowDtype(pyarrow.string()) + if hasattr(pandas, "ArrowDtype") + else pandas.StringDtype() + ), + ), + "arrowdtype_large_string": pandas.Series( + ["m", "n", "o"], + dtype=( + pandas.ArrowDtype(pyarrow.large_string()) + if hasattr(pandas, "ArrowDtype") + and hasattr(pyarrow, "large_string") + else pandas.StringDtype() + ), + ), + }, + ), + expected_df=pandas.DataFrame( + { + "row_num": [1, 2, 3], + "int_want_string": pandas.Series( + ["94043", "10011", "98033"], dtype="object" + ), + "object": pandas.Series(["a", "b", "c"], dtype="object"), + "string_python": pandas.Series(["d", "e", "f"], dtype="object"), + "string_pyarrow": pandas.Series(["g", "h", "i"], dtype="object"), + "arrowdtype_string": pandas.Series(["j", "k", "l"], dtype="object"), + "arrowdtype_large_string": pandas.Series( + ["m", "n", "o"], dtype="object" + ), + }, + ), + table_schema=[ + {"name": "row_num", "type": "INTEGER"}, + {"name": "int_want_string", "type": "STRING"}, + {"name": "object", "type": "STRING"}, + {"name": "string_python", "type": "STRING"}, + {"name": "string_pyarrow", "type": "STRING"}, + {"name": "string_pyarrow_from_int", "type": "STRING"}, + {"name": "arrowdtype_string", "type": "STRING"}, + {"name": "arrowdtype_large_string", "type": "STRING"}, + ], + ), + id="issue875-strings", + ), pytest.param( # Load STRUCT and ARRAY using either object column or ArrowDtype. # See: https://github.com/googleapis/python-bigquery-pandas/issues/452 From e3947191a588b69a32dd9528c974685863af1fc6 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Wed, 5 Feb 2025 14:40:43 -0800 Subject: [PATCH 465/519] chore(python): fix typo in README (#874) Source-Link: https://github.com/googleapis/synthtool/commit/93e1685311a3940e713fd00820aa9937d496f544 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:631b4a35a4f9dd5e97740a97c4c117646eb85b35e103844dc49d152bd18694cd Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 4c0027ff1c61..55a7cb627e68 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,6 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:04c35dc5f49f0f503a306397d6d043685f8d2bb822ab515818c4208d7fb2db3a -# created: 2025-01-16T15:24:11.364245182Z + digest: sha256:631b4a35a4f9dd5e97740a97c4c117646eb85b35e103844dc49d152bd18694cd +# created: 2025-02-05T14:40:56.685429494Z + From b351ede9dba704a6edb05611b9c3d9a957b688a8 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Thu, 6 Feb 2025 10:45:58 -0800 Subject: [PATCH 466/519] chore(main): release 0.27.0 (#877) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> Co-authored-by: Tim Sweña (Swast) --- packages/pandas-gbq/CHANGELOG.md | 7 +++++++ packages/pandas-gbq/pandas_gbq/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index 18fec1f2108e..f628cddd00bd 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.27.0](https://github.com/googleapis/python-bigquery-pandas/compare/v0.26.1...v0.27.0) (2025-02-05) + + +### Features + +* `to_gbq` can write non-string values to existing STRING columns in BigQuery ([#876](https://github.com/googleapis/python-bigquery-pandas/issues/876)) ([ee30a1e](https://github.com/googleapis/python-bigquery-pandas/commit/ee30a1e04a6731275846c04acad6e9646eb4f908)) + ## [0.26.1](https://github.com/googleapis/python-bigquery-pandas/compare/v0.26.0...v0.26.1) (2025-01-06) diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py index 47551e36431c..e9325b3935d9 100644 --- a/packages/pandas-gbq/pandas_gbq/version.py +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -2,4 +2,4 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -__version__ = "0.26.1" +__version__ = "0.27.0" From 4731bf860bd2a547e30177c2ba799bd319d7f438 Mon Sep 17 00:00:00 2001 From: Lingqing Gan Date: Wed, 19 Feb 2025 11:38:20 -0800 Subject: [PATCH 467/519] chore: add sphinx config path to .readthedocs.yml (#881) * chore: add sphinx confit path to .readthedocs.yml * wording --- packages/pandas-gbq/.readthedocs.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/pandas-gbq/.readthedocs.yml b/packages/pandas-gbq/.readthedocs.yml index 7ae21e160a99..86e0f0fb0367 100644 --- a/packages/pandas-gbq/.readthedocs.yml +++ b/packages/pandas-gbq/.readthedocs.yml @@ -16,3 +16,9 @@ build: python: install: - requirements: docs/requirements-docs.txt + +# Explicit configuration path is required by ReadtheDocs starting Jan 20, 2025. +# See: https://about.readthedocs.com/blog/2024/12/deprecate-config-files-without-sphinx-or-mkdocs-config/ +version: 2 +sphinx: + configuration: docs/conf.py From c54db1129e27b441b9d7feba76fe96a28d048939 Mon Sep 17 00:00:00 2001 From: Shenyang Cai Date: Thu, 20 Feb 2025 13:36:32 -0800 Subject: [PATCH 468/519] feat: add bigquery_client as a parameter for read_gbq and to_gbq (#878) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Tim Sweña (Swast) --- packages/pandas-gbq/pandas_gbq/gbq.py | 43 ++++++++++++++++++- packages/pandas-gbq/tests/system/conftest.py | 14 ++++++ packages/pandas-gbq/tests/system/test_gbq.py | 10 +++++ .../pandas-gbq/tests/system/test_read_gbq.py | 11 +++++ .../pandas-gbq/tests/system/test_to_gbq.py | 14 ++++++ 5 files changed, 90 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index feffd8587088..bd3afb973be8 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -269,6 +269,7 @@ def __init__( client_secret=None, user_agent=None, rfc9110_delimiter=False, + bigquery_client=None, ): global context from google.api_core.exceptions import ClientError, GoogleAPIError @@ -288,6 +289,14 @@ def __init__( self.client_secret = client_secret self.user_agent = user_agent self.rfc9110_delimiter = rfc9110_delimiter + self.use_bqstorage_api = use_bqstorage_api + + if bigquery_client is not None: + # If a bq client is already provided, use it to populate auth fields. + self.project_id = bigquery_client.project + self.credentials = bigquery_client._credentials + self.client = bigquery_client + return default_project = None @@ -325,8 +334,9 @@ def __init__( if context.project is None: context.project = self.project_id - self.client = self.get_client() - self.use_bqstorage_api = use_bqstorage_api + self.client = _get_client( + self.user_agent, self.rfc9110_delimiter, self.project_id, self.credentials + ) def _start_timer(self): self.start = time.time() @@ -702,6 +712,7 @@ def read_gbq( client_secret=None, *, col_order=None, + bigquery_client=None, ): r"""Read data from Google BigQuery to a pandas DataFrame. @@ -849,6 +860,9 @@ def read_gbq( the user is attempting to connect to. col_order : list(str), optional Alias for columns, retained for backwards compatibility. + bigquery_client : google.cloud.bigquery.Client, optional + A Google Cloud BigQuery Python Client instance. If provided, it will be used for reading + data, while the project and credentials parameters will be ignored. Returns ------- @@ -900,6 +914,7 @@ def read_gbq( auth_redirect_uri=auth_redirect_uri, client_id=client_id, client_secret=client_secret, + bigquery_client=bigquery_client, ) if _is_query(query_or_table): @@ -971,6 +986,7 @@ def to_gbq( client_secret=None, user_agent=None, rfc9110_delimiter=False, + bigquery_client=None, ): """Write a DataFrame to a Google BigQuery table. @@ -1087,6 +1103,9 @@ def to_gbq( rfc9110_delimiter : bool Sets user agent delimiter to a hyphen or a slash. Default is False, meaning a hyphen will be used. + bigquery_client : google.cloud.bigquery.Client, optional + A Google Cloud BigQuery Python Client instance. If provided, it will be used for reading + data, while the project, user_agent, and credentials parameters will be ignored. .. versionadded:: 0.23.3 """ @@ -1157,6 +1176,7 @@ def to_gbq( client_secret=client_secret, user_agent=user_agent, rfc9110_delimiter=rfc9110_delimiter, + bigquery_client=bigquery_client, ) bqclient = connector.client @@ -1492,3 +1512,22 @@ def create_user_agent( user_agent = f"{user_agent} {identity}" return user_agent + + +def _get_client(user_agent, rfc9110_delimiter, project_id, credentials): + import google.api_core.client_info + + bigquery = FEATURES.bigquery_try_import() + + user_agent = create_user_agent( + user_agent=user_agent, rfc9110_delimiter=rfc9110_delimiter + ) + + client_info = google.api_core.client_info.ClientInfo( + user_agent=user_agent, + ) + return bigquery.Client( + project=project_id, + credentials=credentials, + client_info=client_info, + ) diff --git a/packages/pandas-gbq/tests/system/conftest.py b/packages/pandas-gbq/tests/system/conftest.py index 8c45167fb7af..cb8aadb9c663 100644 --- a/packages/pandas-gbq/tests/system/conftest.py +++ b/packages/pandas-gbq/tests/system/conftest.py @@ -54,6 +54,13 @@ def to_gbq(credentials, project_id): ) +@pytest.fixture +def to_gbq_with_bq_client(bigquery_client): + import pandas_gbq + + return functools.partial(pandas_gbq.to_gbq, bigquery_client=bigquery_client) + + @pytest.fixture def read_gbq(credentials, project_id): import pandas_gbq @@ -63,6 +70,13 @@ def read_gbq(credentials, project_id): ) +@pytest.fixture +def read_gbq_with_bq_client(bigquery_client): + import pandas_gbq + + return functools.partial(pandas_gbq.read_gbq, bigquery_client=bigquery_client) + + @pytest.fixture() def random_dataset_id(bigquery_client: bigquery.Client, project_id: str): dataset_id = prefixer.create_prefix() diff --git a/packages/pandas-gbq/tests/system/test_gbq.py b/packages/pandas-gbq/tests/system/test_gbq.py index b62f35904e7a..1457ec30f0f7 100644 --- a/packages/pandas-gbq/tests/system/test_gbq.py +++ b/packages/pandas-gbq/tests/system/test_gbq.py @@ -1398,3 +1398,13 @@ def test_to_gbq_does_not_override_mode(gbq_table, gbq_connector): ) assert verify_schema(gbq_connector, gbq_table.dataset_id, table_id, table_schema) + + +def test_gbqconnector_init_with_bq_client(bigquery_client): + gbq_connector = gbq.GbqConnector( + project_id="project_id", credentials=None, bigquery_client=bigquery_client + ) + + assert gbq_connector.project_id == bigquery_client.project + assert gbq_connector.credentials is bigquery_client._credentials + assert gbq_connector.client is bigquery_client diff --git a/packages/pandas-gbq/tests/system/test_read_gbq.py b/packages/pandas-gbq/tests/system/test_read_gbq.py index 4ae96a362e2d..72cb6b66ea6a 100644 --- a/packages/pandas-gbq/tests/system/test_read_gbq.py +++ b/packages/pandas-gbq/tests/system/test_read_gbq.py @@ -659,3 +659,14 @@ def test_dml_query(read_gbq, writable_table: str): """ result = read_gbq(query) assert result is not None + + +def test_read_gbq_with_bq_client(read_gbq_with_bq_client): + query = "SELECT * FROM UNNEST([1, 2, 3]) AS numbers" + + actual_result = read_gbq_with_bq_client(query) + + expected_result = pandas.DataFrame( + {"numbers": pandas.Series([1, 2, 3], dtype="Int64")} + ) + pandas.testing.assert_frame_equal(actual_result, expected_result) diff --git a/packages/pandas-gbq/tests/system/test_to_gbq.py b/packages/pandas-gbq/tests/system/test_to_gbq.py index 139f072b6abb..ad7c58ecdd7f 100644 --- a/packages/pandas-gbq/tests/system/test_to_gbq.py +++ b/packages/pandas-gbq/tests/system/test_to_gbq.py @@ -615,3 +615,17 @@ def test_dataframe_round_trip_with_table_schema( pandas.testing.assert_frame_equal( expected_df.set_index("row_num").sort_index(), round_trip ) + + +def test_dataframe_round_trip_with_bq_client( + to_gbq_with_bq_client, read_gbq_with_bq_client, random_dataset_id +): + table_id = ( + f"{random_dataset_id}.round_trip_w_bq_client_{random.randrange(1_000_000)}" + ) + df = pandas.DataFrame({"numbers": pandas.Series([1, 2, 3], dtype="Int64")}) + + to_gbq_with_bq_client(df, table_id) + result = read_gbq_with_bq_client(table_id) + + pandas.testing.assert_frame_equal(result, df) From 9c06b44e0e4efac1e409a258cd35b3168cc33f0d Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Mon, 24 Feb 2025 11:41:17 -0600 Subject: [PATCH 469/519] chore(python): conditionally load credentials in .kokoro/build.sh (#883) Source-Link: https://github.com/googleapis/synthtool/commit/aa69fb74717c8f4c58c60f8cc101d3f4b2c07b09 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:f016446d6e520e5fb552c45b110cba3f217bffdd3d06bdddd076e9e6d13266cf Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 5 ++--- packages/pandas-gbq/.kokoro/build.sh | 20 +++++++++++++------ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 55a7cb627e68..3f7634f25f8e 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,6 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:631b4a35a4f9dd5e97740a97c4c117646eb85b35e103844dc49d152bd18694cd -# created: 2025-02-05T14:40:56.685429494Z - + digest: sha256:f016446d6e520e5fb552c45b110cba3f217bffdd3d06bdddd076e9e6d13266cf +# created: 2025-02-21T19:32:52.01306189Z diff --git a/packages/pandas-gbq/.kokoro/build.sh b/packages/pandas-gbq/.kokoro/build.sh index 08171cbd47a9..d41b45aa1dd0 100755 --- a/packages/pandas-gbq/.kokoro/build.sh +++ b/packages/pandas-gbq/.kokoro/build.sh @@ -15,11 +15,13 @@ set -eo pipefail +CURRENT_DIR=$(dirname "${BASH_SOURCE[0]}") + if [[ -z "${PROJECT_ROOT:-}" ]]; then - PROJECT_ROOT="github/python-bigquery-pandas" + PROJECT_ROOT=$(realpath "${CURRENT_DIR}/..") fi -cd "${PROJECT_ROOT}" +pushd "${PROJECT_ROOT}" # Disable buffering, so that the logs stream through. export PYTHONUNBUFFERED=1 @@ -28,10 +30,16 @@ export PYTHONUNBUFFERED=1 env | grep KOKORO # Setup service account credentials. -export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json +if [[ -f "${KOKORO_GFILE_DIR}/service-account.json" ]] +then + export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json +fi # Setup project id. -export PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json") +if [[ -f "${KOKORO_GFILE_DIR}/project-id.json" ]] +then + export PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json") +fi # If this is a continuous build, send the test log to the FlakyBot. # See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot. @@ -46,7 +54,7 @@ fi # If NOX_SESSION is set, it only runs the specified session, # otherwise run all the sessions. if [[ -n "${NOX_SESSION:-}" ]]; then - python3 -m nox -s ${NOX_SESSION:-} + python3 -m nox -s ${NOX_SESSION:-} else - python3 -m nox + python3 -m nox fi From 90af0df90d17473fef48ff784ceb3f045968581e Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 24 Feb 2025 09:52:12 -0800 Subject: [PATCH 470/519] chore(main): release 0.28.0 (#882) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- packages/pandas-gbq/CHANGELOG.md | 7 +++++++ packages/pandas-gbq/pandas_gbq/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index f628cddd00bd..a860458b1f13 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.28.0](https://github.com/googleapis/python-bigquery-pandas/compare/v0.27.0...v0.28.0) (2025-02-24) + + +### Features + +* Add bigquery_client as a parameter for read_gbq and to_gbq ([#878](https://github.com/googleapis/python-bigquery-pandas/issues/878)) ([d42a562](https://github.com/googleapis/python-bigquery-pandas/commit/d42a56200fe2f356240c7956da4c201e872be4d5)) + ## [0.27.0](https://github.com/googleapis/python-bigquery-pandas/compare/v0.26.1...v0.27.0) (2025-02-05) diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py index e9325b3935d9..a6070398412b 100644 --- a/packages/pandas-gbq/pandas_gbq/version.py +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -2,4 +2,4 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -__version__ = "0.27.0" +__version__ = "0.28.0" From 3c42533a74625909e910d50b32df331965804d98 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Feb 2025 11:11:42 -0800 Subject: [PATCH 471/519] chore(deps): bump cryptography from 43.0.1 to 44.0.1 in /.kokoro (#879) Bumps [cryptography](https://github.com/pyca/cryptography) from 43.0.1 to 44.0.1. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/43.0.1...44.0.1) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Lingqing Gan --- packages/pandas-gbq/.kokoro/requirements.txt | 60 +++++++++++--------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/packages/pandas-gbq/.kokoro/requirements.txt b/packages/pandas-gbq/.kokoro/requirements.txt index 16db448c16bf..6ad95a04a419 100644 --- a/packages/pandas-gbq/.kokoro/requirements.txt +++ b/packages/pandas-gbq/.kokoro/requirements.txt @@ -112,34 +112,38 @@ colorlog==6.8.2 \ # via # gcp-docuploader # nox -cryptography==43.0.1 \ - --hash=sha256:014f58110f53237ace6a408b5beb6c427b64e084eb451ef25a28308270086494 \ - --hash=sha256:1bbcce1a551e262dfbafb6e6252f1ae36a248e615ca44ba302df077a846a8806 \ - --hash=sha256:203e92a75716d8cfb491dc47c79e17d0d9207ccffcbcb35f598fbe463ae3444d \ - --hash=sha256:27e613d7077ac613e399270253259d9d53872aaf657471473ebfc9a52935c062 \ - --hash=sha256:2bd51274dcd59f09dd952afb696bf9c61a7a49dfc764c04dd33ef7a6b502a1e2 \ - --hash=sha256:38926c50cff6f533f8a2dae3d7f19541432610d114a70808f0926d5aaa7121e4 \ - --hash=sha256:511f4273808ab590912a93ddb4e3914dfd8a388fed883361b02dea3791f292e1 \ - --hash=sha256:58d4e9129985185a06d849aa6df265bdd5a74ca6e1b736a77959b498e0505b85 \ - --hash=sha256:5b43d1ea6b378b54a1dc99dd8a2b5be47658fe9a7ce0a58ff0b55f4b43ef2b84 \ - --hash=sha256:61ec41068b7b74268fa86e3e9e12b9f0c21fcf65434571dbb13d954bceb08042 \ - --hash=sha256:666ae11966643886c2987b3b721899d250855718d6d9ce41b521252a17985f4d \ - --hash=sha256:68aaecc4178e90719e95298515979814bda0cbada1256a4485414860bd7ab962 \ - --hash=sha256:7c05650fe8023c5ed0d46793d4b7d7e6cd9c04e68eabe5b0aeea836e37bdcec2 \ - --hash=sha256:80eda8b3e173f0f247f711eef62be51b599b5d425c429b5d4ca6a05e9e856baa \ - --hash=sha256:8385d98f6a3bf8bb2d65a73e17ed87a3ba84f6991c155691c51112075f9ffc5d \ - --hash=sha256:88cce104c36870d70c49c7c8fd22885875d950d9ee6ab54df2745f83ba0dc365 \ - --hash=sha256:9d3cdb25fa98afdd3d0892d132b8d7139e2c087da1712041f6b762e4f807cc96 \ - --hash=sha256:a575913fb06e05e6b4b814d7f7468c2c660e8bb16d8d5a1faf9b33ccc569dd47 \ - --hash=sha256:ac119bb76b9faa00f48128b7f5679e1d8d437365c5d26f1c2c3f0da4ce1b553d \ - --hash=sha256:c1332724be35d23a854994ff0b66530119500b6053d0bd3363265f7e5e77288d \ - --hash=sha256:d03a475165f3134f773d1388aeb19c2d25ba88b6a9733c5c590b9ff7bbfa2e0c \ - --hash=sha256:d75601ad10b059ec832e78823b348bfa1a59f6b8d545db3a24fd44362a1564cb \ - --hash=sha256:de41fd81a41e53267cb020bb3a7212861da53a7d39f863585d13ea11049cf277 \ - --hash=sha256:e710bf40870f4db63c3d7d929aa9e09e4e7ee219e703f949ec4073b4294f6172 \ - --hash=sha256:ea25acb556320250756e53f9e20a4177515f012c9eaea17eb7587a8c4d8ae034 \ - --hash=sha256:f98bf604c82c416bc829e490c700ca1553eafdf2912a91e23a79d97d9801372a \ - --hash=sha256:fba1007b3ef89946dbbb515aeeb41e30203b004f0b4b00e5e16078b518563289 +cryptography==44.0.1 \ + --hash=sha256:00918d859aa4e57db8299607086f793fa7813ae2ff5a4637e318a25ef82730f7 \ + --hash=sha256:1e8d181e90a777b63f3f0caa836844a1182f1f265687fac2115fcf245f5fbec3 \ + --hash=sha256:1f9a92144fa0c877117e9748c74501bea842f93d21ee00b0cf922846d9d0b183 \ + --hash=sha256:21377472ca4ada2906bc313168c9dc7b1d7ca417b63c1c3011d0c74b7de9ae69 \ + --hash=sha256:24979e9f2040c953a94bf3c6782e67795a4c260734e5264dceea65c8f4bae64a \ + --hash=sha256:2a46a89ad3e6176223b632056f321bc7de36b9f9b93b2cc1cccf935a3849dc62 \ + --hash=sha256:322eb03ecc62784536bc173f1483e76747aafeb69c8728df48537eb431cd1911 \ + --hash=sha256:436df4f203482f41aad60ed1813811ac4ab102765ecae7a2bbb1dbb66dcff5a7 \ + --hash=sha256:4f422e8c6a28cf8b7f883eb790695d6d45b0c385a2583073f3cec434cc705e1a \ + --hash=sha256:53f23339864b617a3dfc2b0ac8d5c432625c80014c25caac9082314e9de56f41 \ + --hash=sha256:5fed5cd6102bb4eb843e3315d2bf25fede494509bddadb81e03a859c1bc17b83 \ + --hash=sha256:610a83540765a8d8ce0f351ce42e26e53e1f774a6efb71eb1b41eb01d01c3d12 \ + --hash=sha256:6c8acf6f3d1f47acb2248ec3ea261171a671f3d9428e34ad0357148d492c7864 \ + --hash=sha256:6f76fdd6fd048576a04c5210d53aa04ca34d2ed63336d4abd306d0cbe298fddf \ + --hash=sha256:72198e2b5925155497a5a3e8c216c7fb3e64c16ccee11f0e7da272fa93b35c4c \ + --hash=sha256:887143b9ff6bad2b7570da75a7fe8bbf5f65276365ac259a5d2d5147a73775f2 \ + --hash=sha256:888fcc3fce0c888785a4876ca55f9f43787f4c5c1cc1e2e0da71ad481ff82c5b \ + --hash=sha256:8e6a85a93d0642bd774460a86513c5d9d80b5c002ca9693e63f6e540f1815ed0 \ + --hash=sha256:94f99f2b943b354a5b6307d7e8d19f5c423a794462bde2bf310c770ba052b1c4 \ + --hash=sha256:9b336599e2cb77b1008cb2ac264b290803ec5e8e89d618a5e978ff5eb6f715d9 \ + --hash=sha256:a2d8a7045e1ab9b9f803f0d9531ead85f90c5f2859e653b61497228b18452008 \ + --hash=sha256:b8272f257cf1cbd3f2e120f14c68bff2b6bdfcc157fafdee84a1b795efd72862 \ + --hash=sha256:bf688f615c29bfe9dfc44312ca470989279f0e94bb9f631f85e3459af8efc009 \ + --hash=sha256:d9c5b9f698a83c8bd71e0f4d3f9f839ef244798e5ffe96febfa9714717db7af7 \ + --hash=sha256:dd7c7e2d71d908dc0f8d2027e1604102140d84b155e658c20e8ad1304317691f \ + --hash=sha256:df978682c1504fc93b3209de21aeabf2375cb1571d4e61907b3e7a2540e83026 \ + --hash=sha256:e403f7f766ded778ecdb790da786b418a9f2394f36e8cc8b796cc056ab05f44f \ + --hash=sha256:eb3889330f2a4a148abead555399ec9a32b13b7c8ba969b72d8e500eb7ef84cd \ + --hash=sha256:f4daefc971c2d1f82f03097dc6f216744a6cd2ac0f04c68fb935ea2ba2a0d420 \ + --hash=sha256:f51f5705ab27898afda1aaa430f34ad90dc117421057782022edf0600bec5f14 \ + --hash=sha256:fd0ee90072861e276b0ff08bd627abec29e32a53b2be44e41dbcdf87cbee2b00 # via # -r requirements.in # gcp-releasetool From 250fedab8272215e5f9f8a748cc52cbbc316ea26 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Mon, 24 Feb 2025 20:32:52 +0100 Subject: [PATCH 472/519] chore(deps): update all dependencies (#853) Co-authored-by: Lingqing Gan --- .../.kokoro/docker/docs/requirements.txt | 66 +++++++++---------- .../samples/snippets/requirements-test.txt | 2 +- .../samples/snippets/requirements.txt | 8 +-- 3 files changed, 38 insertions(+), 38 deletions(-) diff --git a/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt b/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt index a9360a25b707..d0bcc1176845 100644 --- a/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt +++ b/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt @@ -8,13 +8,13 @@ argcomplete==3.5.3 \ --hash=sha256:2ab2c4a215c59fd6caaff41a869480a23e8f6a5f910b266c1808037f4e375b61 \ --hash=sha256:c12bf50eded8aebb298c7b7da7a5ff3ee24dffd9f5281867dfe1424b58c55392 # via nox -cachetools==5.5.0 \ - --hash=sha256:02134e8439cdc2ffb62023ce1debca2944c3f289d66bb17ead3ab3dede74b292 \ - --hash=sha256:2cc24fb4cbe39633fb7badd9db9ca6295d766d9c2995f245725a46715d050f2a +cachetools==5.5.2 \ + --hash=sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4 \ + --hash=sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a # via google-auth -certifi==2024.12.14 \ - --hash=sha256:1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56 \ - --hash=sha256:b650d30f370c2b724812bee08008be0c4163b163ddaec3f2546c1caf65f191db +certifi==2025.1.31 \ + --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ + --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe # via requests charset-normalizer==3.4.1 \ --hash=sha256:0167ddc8ab6508fe81860a57dd472b2ef4060e8d378f0cc555707126830f2537 \ @@ -124,34 +124,34 @@ distlib==0.3.9 \ --hash=sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87 \ --hash=sha256:a60f20dea646b8a33f3e7772f74dc0b2d0772d2837ee1342a00645c81edf9403 # via virtualenv -filelock==3.16.1 \ - --hash=sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0 \ - --hash=sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435 +filelock==3.17.0 \ + --hash=sha256:533dc2f7ba78dc2f0f531fc6c4940addf7b70a481e269a5a3b93be94ffbe8338 \ + --hash=sha256:ee4e77401ef576ebb38cd7f13b9b28893194acc20a8e68e18730ba9c0e54660e # via virtualenv gcp-docuploader==0.6.5 \ --hash=sha256:30221d4ac3e5a2b9c69aa52fdbef68cc3f27d0e6d0d90e220fc024584b8d2318 \ --hash=sha256:b7458ef93f605b9d46a4bf3a8dc1755dad1f31d030c8679edf304e343b347eea # via -r requirements.in -google-api-core==2.24.0 \ - --hash=sha256:10d82ac0fca69c82a25b3efdeefccf6f28e02ebb97925a8cce8edbfe379929d9 \ - --hash=sha256:e255640547a597a4da010876d333208ddac417d60add22b6851a0c66a831fcaf +google-api-core==2.24.1 \ + --hash=sha256:bc78d608f5a5bf853b80bd70a795f703294de656c096c0968320830a4bc280f1 \ + --hash=sha256:f8b36f5456ab0dd99a1b693a40a31d1e7757beea380ad1b38faaf8941eae9d8a # via # google-cloud-core # google-cloud-storage -google-auth==2.37.0 \ - --hash=sha256:0054623abf1f9c83492c63d3f47e77f0a544caa3d40b2d98e099a611c2dd5d00 \ - --hash=sha256:42664f18290a6be591be5329a96fe30184be1a1badb7292a7f686a9659de9ca0 +google-auth==2.38.0 \ + --hash=sha256:8285113607d3b80a3f1543b75962447ba8a09fe85783432a784fdeef6ac094c4 \ + --hash=sha256:e7dae6694313f434a2727bf2906f27ad259bae090d7aa896590d86feec3d9d4a # via # google-api-core # google-cloud-core # google-cloud-storage -google-cloud-core==2.4.1 \ - --hash=sha256:9b7749272a812bde58fff28868d0c5e2f585b82f37e09a1f6ed2d4d10f134073 \ - --hash=sha256:a9e6a4422b9ac5c29f79a0ede9485473338e2ce78d91f2370c01e730eab22e61 +google-cloud-core==2.4.2 \ + --hash=sha256:7459c3e83de7cb8b9ecfec9babc910efb4314030c56dd798eaad12c426f7d180 \ + --hash=sha256:a4fcb0e2fcfd4bfe963837fad6d10943754fd79c1a50097d68540b6eb3d67f35 # via google-cloud-storage -google-cloud-storage==2.19.0 \ - --hash=sha256:aeb971b5c29cf8ab98445082cbfe7b161a1f48ed275822f59ed3f1524ea54fba \ - --hash=sha256:cd05e9e7191ba6cb68934d8eb76054d9be4562aa89dbc4236feee4d7d51342b2 +google-cloud-storage==3.0.0 \ + --hash=sha256:2accb3e828e584888beff1165e5f3ac61aa9088965eb0165794a82d8c7f95297 \ + --hash=sha256:f85fd059650d2dbb0ac158a9a6b304b66143b35ed2419afec2905ca522eb2c6a # via gcp-docuploader google-crc32c==1.6.0 \ --hash=sha256:05e2d8c9a2f853ff116db9706b4a27350587f341eda835f46db3c0a8c8ce2f24 \ @@ -188,17 +188,17 @@ google-resumable-media==2.7.2 \ --hash=sha256:3ce7551e9fe6d99e9a126101d2536612bb73486721951e9562fee0f90c6ababa \ --hash=sha256:5280aed4629f2b60b847b0d42f9857fd4935c11af266744df33d8074cae92fe0 # via google-cloud-storage -googleapis-common-protos==1.66.0 \ - --hash=sha256:c3e7b33d15fdca5374cc0a7346dd92ffa847425cc4ea941d970f13680052ec8c \ - --hash=sha256:d7abcd75fabb2e0ec9f74466401f6c119a0b498e27370e9be4c94cb7e382b8ed +googleapis-common-protos==1.68.0 \ + --hash=sha256:95d38161f4f9af0d9423eed8fb7b64ffd2568c3464eb542ff02c5bfa1953ab3c \ + --hash=sha256:aaf179b2f81df26dfadac95def3b16a95064c76a5f45f07e4c68a21bb371c4ac # via google-api-core idna==3.10 \ --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \ --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3 # via requests -nox==2024.10.9 \ - --hash=sha256:1d36f309a0a2a853e9bccb76bbef6bb118ba92fa92674d15604ca99adeb29eab \ - --hash=sha256:7aa9dc8d1c27e9f45ab046ffd1c3b2c4f7c91755304769df231308849ebded95 +nox==2025.2.9 \ + --hash=sha256:7d1e92d1918c6980d70aee9cf1c1d19d16faa71c4afe338fffd39e8a460e2067 \ + --hash=sha256:d50cd4ca568bd7621c2e6cbbc4845b3b7f7697f25d5fb0190ce8f4600be79768 # via -r requirements.in packaging==24.2 \ --hash=sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759 \ @@ -208,9 +208,9 @@ platformdirs==4.3.6 \ --hash=sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907 \ --hash=sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb # via virtualenv -proto-plus==1.25.0 \ - --hash=sha256:c91fc4a65074ade8e458e95ef8bac34d4008daa7cce4a12d6707066fca648961 \ - --hash=sha256:fbb17f57f7bd05a68b7707e745e26528b0b3c34e378db91eef93912c54982d91 +proto-plus==1.26.0 \ + --hash=sha256:6e93d5f5ca267b54300880fff156b6a3386b3fa3f43b1da62e680fc0c586ef22 \ + --hash=sha256:bf2dfaa3da281fc3187d12d224c707cb57214fb2c22ba854eb0c105a3fb2d4d7 # via google-api-core protobuf==5.29.3 \ --hash=sha256:0a18ed4a24198528f2333802eb075e59dea9d679ab7a6c5efb017a59004d849f \ @@ -291,7 +291,7 @@ urllib3==2.3.0 \ --hash=sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df \ --hash=sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d # via requests -virtualenv==20.28.1 \ - --hash=sha256:412773c85d4dab0409b83ec36f7a6499e72eaf08c80e81e9576bca61831c71cb \ - --hash=sha256:5d34ab240fdb5d21549b76f9e8ff3af28252f5499fb6d6f031adac4e5a8c5329 +virtualenv==20.29.2 \ + --hash=sha256:fdaabebf6d03b5ba83ae0a02cfe96f48a716f4fae556461d180825866f75b728 \ + --hash=sha256:febddfc3d1ea571bdb1dc0f98d7b45d24def7428214d4fb73cc486c9568cce6a # via nox diff --git a/packages/pandas-gbq/samples/snippets/requirements-test.txt b/packages/pandas-gbq/samples/snippets/requirements-test.txt index 48d1a2e4fe74..c8ce62ede005 100644 --- a/packages/pandas-gbq/samples/snippets/requirements-test.txt +++ b/packages/pandas-gbq/samples/snippets/requirements-test.txt @@ -1,2 +1,2 @@ -google-cloud-testutils==1.5.0 +google-cloud-testutils==1.6.0 pytest==8.3.4 diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 2bfdd4a1ba43..d8bf9ea99981 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,6 +1,6 @@ -google-cloud-bigquery-storage==2.27.0 -google-cloud-bigquery==3.27.0 -pandas-gbq==0.26.1 +google-cloud-bigquery-storage==2.28.0 +google-cloud-bigquery==3.29.0 +pandas-gbq==0.27.0 pandas===2.0.3; python_version == '3.8' pandas==2.2.3; python_version >= '3.9' -pyarrow==18.1.0; python_version >= '3.9' +pyarrow==19.0.1; python_version >= '3.9' From de6fcd13a956de1f74b62654a1619ae09ed2e89c Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Mon, 24 Feb 2025 22:31:42 +0100 Subject: [PATCH 473/519] chore(deps): update dependency pandas-gbq to v0.28.0 (#884) --- packages/pandas-gbq/samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index d8bf9ea99981..731aecbcd997 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,6 +1,6 @@ google-cloud-bigquery-storage==2.28.0 google-cloud-bigquery==3.29.0 -pandas-gbq==0.27.0 +pandas-gbq==0.28.0 pandas===2.0.3; python_version == '3.8' pandas==2.2.3; python_version >= '3.9' pyarrow==19.0.1; python_version >= '3.9' From e70937c27bf43f50da929b6e57cc3178207834e8 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Fri, 28 Feb 2025 21:37:28 +0100 Subject: [PATCH 474/519] chore(deps): update all dependencies (#885) --- packages/pandas-gbq/.kokoro/docker/docs/requirements.txt | 6 +++--- packages/pandas-gbq/samples/snippets/requirements.txt | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt b/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt index d0bcc1176845..9634ff2f676e 100644 --- a/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt +++ b/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt @@ -149,9 +149,9 @@ google-cloud-core==2.4.2 \ --hash=sha256:7459c3e83de7cb8b9ecfec9babc910efb4314030c56dd798eaad12c426f7d180 \ --hash=sha256:a4fcb0e2fcfd4bfe963837fad6d10943754fd79c1a50097d68540b6eb3d67f35 # via google-cloud-storage -google-cloud-storage==3.0.0 \ - --hash=sha256:2accb3e828e584888beff1165e5f3ac61aa9088965eb0165794a82d8c7f95297 \ - --hash=sha256:f85fd059650d2dbb0ac158a9a6b304b66143b35ed2419afec2905ca522eb2c6a +google-cloud-storage==3.1.0 \ + --hash=sha256:944273179897c7c8a07ee15f2e6466a02da0c7c4b9ecceac2a26017cb2972049 \ + --hash=sha256:eaf36966b68660a9633f03b067e4a10ce09f1377cae3ff9f2c699f69a81c66c6 # via gcp-docuploader google-crc32c==1.6.0 \ --hash=sha256:05e2d8c9a2f853ff116db9706b4a27350587f341eda835f46db3c0a8c8ce2f24 \ diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 731aecbcd997..8e41d443d2b1 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,5 +1,5 @@ google-cloud-bigquery-storage==2.28.0 -google-cloud-bigquery==3.29.0 +google-cloud-bigquery==3.30.0 pandas-gbq==0.28.0 pandas===2.0.3; python_version == '3.8' pandas==2.2.3; python_version >= '3.9' From 1aa5b9803c8cac3451e193c6caf732641e787821 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Tue, 4 Mar 2025 20:55:36 +0100 Subject: [PATCH 475/519] chore(deps): update all dependencies (#887) --- packages/pandas-gbq/.kokoro/docker/docs/requirements.txt | 6 +++--- packages/pandas-gbq/samples/snippets/requirements-test.txt | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt b/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt index 9634ff2f676e..af7cdeedfb14 100644 --- a/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt +++ b/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt @@ -188,9 +188,9 @@ google-resumable-media==2.7.2 \ --hash=sha256:3ce7551e9fe6d99e9a126101d2536612bb73486721951e9562fee0f90c6ababa \ --hash=sha256:5280aed4629f2b60b847b0d42f9857fd4935c11af266744df33d8074cae92fe0 # via google-cloud-storage -googleapis-common-protos==1.68.0 \ - --hash=sha256:95d38161f4f9af0d9423eed8fb7b64ffd2568c3464eb542ff02c5bfa1953ab3c \ - --hash=sha256:aaf179b2f81df26dfadac95def3b16a95064c76a5f45f07e4c68a21bb371c4ac +googleapis-common-protos==1.69.0 \ + --hash=sha256:17835fdc4fa8da1d61cfe2d4d5d57becf7c61d4112f8d81c67eaa9d7ce43042d \ + --hash=sha256:5a46d58af72846f59009b9c4710425b9af2139555c71837081706b213b298187 # via google-api-core idna==3.10 \ --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \ diff --git a/packages/pandas-gbq/samples/snippets/requirements-test.txt b/packages/pandas-gbq/samples/snippets/requirements-test.txt index c8ce62ede005..417ef03776ef 100644 --- a/packages/pandas-gbq/samples/snippets/requirements-test.txt +++ b/packages/pandas-gbq/samples/snippets/requirements-test.txt @@ -1,2 +1,2 @@ google-cloud-testutils==1.6.0 -pytest==8.3.4 +pytest==8.3.5 From 1b2e33595d36b8c556a96d68292125b1a0f8a526 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 6 Mar 2025 10:35:22 -0500 Subject: [PATCH 476/519] chore: remove unused files (#889) --- packages/pandas-gbq/.github/.OwlBot.lock.yaml | 4 +- .../pandas-gbq/.kokoro/docker/docs/Dockerfile | 89 --- .../.kokoro/docker/docs/requirements.in | 2 - .../.kokoro/docker/docs/requirements.txt | 297 ---------- packages/pandas-gbq/.kokoro/docs/common.cfg | 67 --- .../.kokoro/docs/docs-presubmit.cfg | 28 - packages/pandas-gbq/.kokoro/docs/docs.cfg | 1 - packages/pandas-gbq/.kokoro/publish-docs.sh | 58 -- packages/pandas-gbq/.kokoro/release.sh | 29 - .../pandas-gbq/.kokoro/release/common.cfg | 43 -- .../pandas-gbq/.kokoro/release/release.cfg | 1 - packages/pandas-gbq/.kokoro/requirements.in | 11 - packages/pandas-gbq/.kokoro/requirements.txt | 513 ------------------ 13 files changed, 2 insertions(+), 1141 deletions(-) delete mode 100644 packages/pandas-gbq/.kokoro/docker/docs/Dockerfile delete mode 100644 packages/pandas-gbq/.kokoro/docker/docs/requirements.in delete mode 100644 packages/pandas-gbq/.kokoro/docker/docs/requirements.txt delete mode 100644 packages/pandas-gbq/.kokoro/docs/common.cfg delete mode 100644 packages/pandas-gbq/.kokoro/docs/docs-presubmit.cfg delete mode 100644 packages/pandas-gbq/.kokoro/docs/docs.cfg delete mode 100755 packages/pandas-gbq/.kokoro/publish-docs.sh delete mode 100755 packages/pandas-gbq/.kokoro/release.sh delete mode 100644 packages/pandas-gbq/.kokoro/release/common.cfg delete mode 100644 packages/pandas-gbq/.kokoro/release/release.cfg delete mode 100644 packages/pandas-gbq/.kokoro/requirements.in delete mode 100644 packages/pandas-gbq/.kokoro/requirements.txt diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml index 3f7634f25f8e..c631e1f7d7e9 100644 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ b/packages/pandas-gbq/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:f016446d6e520e5fb552c45b110cba3f217bffdd3d06bdddd076e9e6d13266cf -# created: 2025-02-21T19:32:52.01306189Z + digest: sha256:5581906b957284864632cde4e9c51d1cc66b0094990b27e689132fe5cd036046 +# created: 2025-03-05 diff --git a/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile b/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile deleted file mode 100644 index e5410e296bd8..000000000000 --- a/packages/pandas-gbq/.kokoro/docker/docs/Dockerfile +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ubuntu:24.04 - -ENV DEBIAN_FRONTEND noninteractive - -# Ensure local Python is preferred over distribution Python. -ENV PATH /usr/local/bin:$PATH - -# Install dependencies. -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - apt-transport-https \ - build-essential \ - ca-certificates \ - curl \ - dirmngr \ - git \ - gpg-agent \ - graphviz \ - libbz2-dev \ - libdb5.3-dev \ - libexpat1-dev \ - libffi-dev \ - liblzma-dev \ - libreadline-dev \ - libsnappy-dev \ - libssl-dev \ - libsqlite3-dev \ - portaudio19-dev \ - redis-server \ - software-properties-common \ - ssh \ - sudo \ - tcl \ - tcl-dev \ - tk \ - tk-dev \ - uuid-dev \ - wget \ - zlib1g-dev \ - && add-apt-repository universe \ - && apt-get update \ - && apt-get -y install jq \ - && apt-get clean autoclean \ - && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* \ - && rm -f /var/cache/apt/archives/*.deb - - -###################### Install python 3.10.14 for docs/docfx session - -# Download python 3.10.14 -RUN wget https://www.python.org/ftp/python/3.10.14/Python-3.10.14.tgz - -# Extract files -RUN tar -xvf Python-3.10.14.tgz - -# Install python 3.10.14 -RUN ./Python-3.10.14/configure --enable-optimizations -RUN make altinstall - -ENV PATH /usr/local/bin/python3.10:$PATH - -###################### Install pip -RUN wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \ - && python3.10 /tmp/get-pip.py \ - && rm /tmp/get-pip.py - -# Test pip -RUN python3.10 -m pip - -# Install build requirements -COPY requirements.txt /requirements.txt -RUN python3.10 -m pip install --require-hashes -r requirements.txt - -CMD ["python3.10"] diff --git a/packages/pandas-gbq/.kokoro/docker/docs/requirements.in b/packages/pandas-gbq/.kokoro/docker/docs/requirements.in deleted file mode 100644 index 586bd07037ae..000000000000 --- a/packages/pandas-gbq/.kokoro/docker/docs/requirements.in +++ /dev/null @@ -1,2 +0,0 @@ -nox -gcp-docuploader diff --git a/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt b/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt deleted file mode 100644 index af7cdeedfb14..000000000000 --- a/packages/pandas-gbq/.kokoro/docker/docs/requirements.txt +++ /dev/null @@ -1,297 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile --allow-unsafe --generate-hashes requirements.in -# -argcomplete==3.5.3 \ - --hash=sha256:2ab2c4a215c59fd6caaff41a869480a23e8f6a5f910b266c1808037f4e375b61 \ - --hash=sha256:c12bf50eded8aebb298c7b7da7a5ff3ee24dffd9f5281867dfe1424b58c55392 - # via nox -cachetools==5.5.2 \ - --hash=sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4 \ - --hash=sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a - # via google-auth -certifi==2025.1.31 \ - --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ - --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe - # via requests -charset-normalizer==3.4.1 \ - --hash=sha256:0167ddc8ab6508fe81860a57dd472b2ef4060e8d378f0cc555707126830f2537 \ - --hash=sha256:01732659ba9b5b873fc117534143e4feefecf3b2078b0a6a2e925271bb6f4cfa \ - --hash=sha256:01ad647cdd609225c5350561d084b42ddf732f4eeefe6e678765636791e78b9a \ - --hash=sha256:04432ad9479fa40ec0f387795ddad4437a2b50417c69fa275e212933519ff294 \ - --hash=sha256:0907f11d019260cdc3f94fbdb23ff9125f6b5d1039b76003b5b0ac9d6a6c9d5b \ - --hash=sha256:0924e81d3d5e70f8126529951dac65c1010cdf117bb75eb02dd12339b57749dd \ - --hash=sha256:09b26ae6b1abf0d27570633b2b078a2a20419c99d66fb2823173d73f188ce601 \ - --hash=sha256:09b5e6733cbd160dcc09589227187e242a30a49ca5cefa5a7edd3f9d19ed53fd \ - --hash=sha256:0af291f4fe114be0280cdd29d533696a77b5b49cfde5467176ecab32353395c4 \ - --hash=sha256:0f55e69f030f7163dffe9fd0752b32f070566451afe180f99dbeeb81f511ad8d \ - --hash=sha256:1a2bc9f351a75ef49d664206d51f8e5ede9da246602dc2d2726837620ea034b2 \ - --hash=sha256:22e14b5d70560b8dd51ec22863f370d1e595ac3d024cb8ad7d308b4cd95f8313 \ - --hash=sha256:234ac59ea147c59ee4da87a0c0f098e9c8d169f4dc2a159ef720f1a61bbe27cd \ - --hash=sha256:2369eea1ee4a7610a860d88f268eb39b95cb588acd7235e02fd5a5601773d4fa \ - --hash=sha256:237bdbe6159cff53b4f24f397d43c6336c6b0b42affbe857970cefbb620911c8 \ - --hash=sha256:28bf57629c75e810b6ae989f03c0828d64d6b26a5e205535585f96093e405ed1 \ - --hash=sha256:2967f74ad52c3b98de4c3b32e1a44e32975e008a9cd2a8cc8966d6a5218c5cb2 \ - --hash=sha256:2a75d49014d118e4198bcee5ee0a6f25856b29b12dbf7cd012791f8a6cc5c496 \ - --hash=sha256:2bdfe3ac2e1bbe5b59a1a63721eb3b95fc9b6817ae4a46debbb4e11f6232428d \ - --hash=sha256:2d074908e1aecee37a7635990b2c6d504cd4766c7bc9fc86d63f9c09af3fa11b \ - --hash=sha256:2fb9bd477fdea8684f78791a6de97a953c51831ee2981f8e4f583ff3b9d9687e \ - --hash=sha256:311f30128d7d333eebd7896965bfcfbd0065f1716ec92bd5638d7748eb6f936a \ - --hash=sha256:329ce159e82018d646c7ac45b01a430369d526569ec08516081727a20e9e4af4 \ - --hash=sha256:345b0426edd4e18138d6528aed636de7a9ed169b4aaf9d61a8c19e39d26838ca \ - --hash=sha256:363e2f92b0f0174b2f8238240a1a30142e3db7b957a5dd5689b0e75fb717cc78 \ - --hash=sha256:3a3bd0dcd373514dcec91c411ddb9632c0d7d92aed7093b8c3bbb6d69ca74408 \ - --hash=sha256:3bed14e9c89dcb10e8f3a29f9ccac4955aebe93c71ae803af79265c9ca5644c5 \ - --hash=sha256:44251f18cd68a75b56585dd00dae26183e102cd5e0f9f1466e6df5da2ed64ea3 \ - --hash=sha256:44ecbf16649486d4aebafeaa7ec4c9fed8b88101f4dd612dcaf65d5e815f837f \ - --hash=sha256:4532bff1b8421fd0a320463030c7520f56a79c9024a4e88f01c537316019005a \ - --hash=sha256:49402233c892a461407c512a19435d1ce275543138294f7ef013f0b63d5d3765 \ - --hash=sha256:4c0907b1928a36d5a998d72d64d8eaa7244989f7aaaf947500d3a800c83a3fd6 \ - --hash=sha256:4d86f7aff21ee58f26dcf5ae81a9addbd914115cdebcbb2217e4f0ed8982e146 \ - --hash=sha256:5777ee0881f9499ed0f71cc82cf873d9a0ca8af166dfa0af8ec4e675b7df48e6 \ - --hash=sha256:5df196eb874dae23dcfb968c83d4f8fdccb333330fe1fc278ac5ceeb101003a9 \ - --hash=sha256:619a609aa74ae43d90ed2e89bdd784765de0a25ca761b93e196d938b8fd1dbbd \ - --hash=sha256:6e27f48bcd0957c6d4cb9d6fa6b61d192d0b13d5ef563e5f2ae35feafc0d179c \ - --hash=sha256:6ff8a4a60c227ad87030d76e99cd1698345d4491638dfa6673027c48b3cd395f \ - --hash=sha256:73d94b58ec7fecbc7366247d3b0b10a21681004153238750bb67bd9012414545 \ - --hash=sha256:7461baadb4dc00fd9e0acbe254e3d7d2112e7f92ced2adc96e54ef6501c5f176 \ - --hash=sha256:75832c08354f595c760a804588b9357d34ec00ba1c940c15e31e96d902093770 \ - --hash=sha256:7709f51f5f7c853f0fb938bcd3bc59cdfdc5203635ffd18bf354f6967ea0f824 \ - --hash=sha256:78baa6d91634dfb69ec52a463534bc0df05dbd546209b79a3880a34487f4b84f \ - --hash=sha256:7974a0b5ecd505609e3b19742b60cee7aa2aa2fb3151bc917e6e2646d7667dcf \ - --hash=sha256:7a4f97a081603d2050bfaffdefa5b02a9ec823f8348a572e39032caa8404a487 \ - --hash=sha256:7b1bef6280950ee6c177b326508f86cad7ad4dff12454483b51d8b7d673a2c5d \ - --hash=sha256:7d053096f67cd1241601111b698f5cad775f97ab25d81567d3f59219b5f1adbd \ - --hash=sha256:804a4d582ba6e5b747c625bf1255e6b1507465494a40a2130978bda7b932c90b \ - --hash=sha256:807f52c1f798eef6cf26beb819eeb8819b1622ddfeef9d0977a8502d4db6d534 \ - --hash=sha256:80ed5e856eb7f30115aaf94e4a08114ccc8813e6ed1b5efa74f9f82e8509858f \ - --hash=sha256:8417cb1f36cc0bc7eaba8ccb0e04d55f0ee52df06df3ad55259b9a323555fc8b \ - --hash=sha256:8436c508b408b82d87dc5f62496973a1805cd46727c34440b0d29d8a2f50a6c9 \ - --hash=sha256:89149166622f4db9b4b6a449256291dc87a99ee53151c74cbd82a53c8c2f6ccd \ - --hash=sha256:8bfa33f4f2672964266e940dd22a195989ba31669bd84629f05fab3ef4e2d125 \ - --hash=sha256:8c60ca7339acd497a55b0ea5d506b2a2612afb2826560416f6894e8b5770d4a9 \ - --hash=sha256:91b36a978b5ae0ee86c394f5a54d6ef44db1de0815eb43de826d41d21e4af3de \ - --hash=sha256:955f8851919303c92343d2f66165294848d57e9bba6cf6e3625485a70a038d11 \ - --hash=sha256:97f68b8d6831127e4787ad15e6757232e14e12060bec17091b85eb1486b91d8d \ - --hash=sha256:9b23ca7ef998bc739bf6ffc077c2116917eabcc901f88da1b9856b210ef63f35 \ - --hash=sha256:9f0b8b1c6d84c8034a44893aba5e767bf9c7a211e313a9605d9c617d7083829f \ - --hash=sha256:aabfa34badd18f1da5ec1bc2715cadc8dca465868a4e73a0173466b688f29dda \ - --hash=sha256:ab36c8eb7e454e34e60eb55ca5d241a5d18b2c6244f6827a30e451c42410b5f7 \ - --hash=sha256:b010a7a4fd316c3c484d482922d13044979e78d1861f0e0650423144c616a46a \ - --hash=sha256:b1ac5992a838106edb89654e0aebfc24f5848ae2547d22c2c3f66454daa11971 \ - --hash=sha256:b7b2d86dd06bfc2ade3312a83a5c364c7ec2e3498f8734282c6c3d4b07b346b8 \ - --hash=sha256:b97e690a2118911e39b4042088092771b4ae3fc3aa86518f84b8cf6888dbdb41 \ - --hash=sha256:bc2722592d8998c870fa4e290c2eec2c1569b87fe58618e67d38b4665dfa680d \ - --hash=sha256:c0429126cf75e16c4f0ad00ee0eae4242dc652290f940152ca8c75c3a4b6ee8f \ - --hash=sha256:c30197aa96e8eed02200a83fba2657b4c3acd0f0aa4bdc9f6c1af8e8962e0757 \ - --hash=sha256:c4c3e6da02df6fa1410a7680bd3f63d4f710232d3139089536310d027950696a \ - --hash=sha256:c75cb2a3e389853835e84a2d8fb2b81a10645b503eca9bcb98df6b5a43eb8886 \ - --hash=sha256:c96836c97b1238e9c9e3fe90844c947d5afbf4f4c92762679acfe19927d81d77 \ - --hash=sha256:d7f50a1f8c450f3925cb367d011448c39239bb3eb4117c36a6d354794de4ce76 \ - --hash=sha256:d973f03c0cb71c5ed99037b870f2be986c3c05e63622c017ea9816881d2dd247 \ - --hash=sha256:d98b1668f06378c6dbefec3b92299716b931cd4e6061f3c875a71ced1780ab85 \ - --hash=sha256:d9c3cdf5390dcd29aa8056d13e8e99526cda0305acc038b96b30352aff5ff2bb \ - --hash=sha256:dad3e487649f498dd991eeb901125411559b22e8d7ab25d3aeb1af367df5efd7 \ - --hash=sha256:dccbe65bd2f7f7ec22c4ff99ed56faa1e9f785482b9bbd7c717e26fd723a1d1e \ - --hash=sha256:dd78cfcda14a1ef52584dbb008f7ac81c1328c0f58184bf9a84c49c605002da6 \ - --hash=sha256:e218488cd232553829be0664c2292d3af2eeeb94b32bea483cf79ac6a694e037 \ - --hash=sha256:e358e64305fe12299a08e08978f51fc21fac060dcfcddd95453eabe5b93ed0e1 \ - --hash=sha256:ea0d8d539afa5eb2728aa1932a988a9a7af94f18582ffae4bc10b3fbdad0626e \ - --hash=sha256:eab677309cdb30d047996b36d34caeda1dc91149e4fdca0b1a039b3f79d9a807 \ - --hash=sha256:eb8178fe3dba6450a3e024e95ac49ed3400e506fd4e9e5c32d30adda88cbd407 \ - --hash=sha256:ecddf25bee22fe4fe3737a399d0d177d72bc22be6913acfab364b40bce1ba83c \ - --hash=sha256:eea6ee1db730b3483adf394ea72f808b6e18cf3cb6454b4d86e04fa8c4327a12 \ - --hash=sha256:f08ff5e948271dc7e18a35641d2f11a4cd8dfd5634f55228b691e62b37125eb3 \ - --hash=sha256:f30bf9fd9be89ecb2360c7d94a711f00c09b976258846efe40db3d05828e8089 \ - --hash=sha256:fa88b843d6e211393a37219e6a1c1df99d35e8fd90446f1118f4216e307e48cd \ - --hash=sha256:fc54db6c8593ef7d4b2a331b58653356cf04f67c960f584edb7c3d8c97e8f39e \ - --hash=sha256:fd4ec41f914fa74ad1b8304bbc634b3de73d2a0889bd32076342a573e0779e00 \ - --hash=sha256:ffc9202a29ab3920fa812879e95a9e78b2465fd10be7fcbd042899695d75e616 - # via requests -click==8.1.8 \ - --hash=sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2 \ - --hash=sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a - # via gcp-docuploader -colorlog==6.9.0 \ - --hash=sha256:5906e71acd67cb07a71e779c47c4bcb45fb8c2993eebe9e5adcd6a6f1b283eff \ - --hash=sha256:bfba54a1b93b94f54e1f4fe48395725a3d92fd2a4af702f6bd70946bdc0c6ac2 - # via - # gcp-docuploader - # nox -distlib==0.3.9 \ - --hash=sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87 \ - --hash=sha256:a60f20dea646b8a33f3e7772f74dc0b2d0772d2837ee1342a00645c81edf9403 - # via virtualenv -filelock==3.17.0 \ - --hash=sha256:533dc2f7ba78dc2f0f531fc6c4940addf7b70a481e269a5a3b93be94ffbe8338 \ - --hash=sha256:ee4e77401ef576ebb38cd7f13b9b28893194acc20a8e68e18730ba9c0e54660e - # via virtualenv -gcp-docuploader==0.6.5 \ - --hash=sha256:30221d4ac3e5a2b9c69aa52fdbef68cc3f27d0e6d0d90e220fc024584b8d2318 \ - --hash=sha256:b7458ef93f605b9d46a4bf3a8dc1755dad1f31d030c8679edf304e343b347eea - # via -r requirements.in -google-api-core==2.24.1 \ - --hash=sha256:bc78d608f5a5bf853b80bd70a795f703294de656c096c0968320830a4bc280f1 \ - --hash=sha256:f8b36f5456ab0dd99a1b693a40a31d1e7757beea380ad1b38faaf8941eae9d8a - # via - # google-cloud-core - # google-cloud-storage -google-auth==2.38.0 \ - --hash=sha256:8285113607d3b80a3f1543b75962447ba8a09fe85783432a784fdeef6ac094c4 \ - --hash=sha256:e7dae6694313f434a2727bf2906f27ad259bae090d7aa896590d86feec3d9d4a - # via - # google-api-core - # google-cloud-core - # google-cloud-storage -google-cloud-core==2.4.2 \ - --hash=sha256:7459c3e83de7cb8b9ecfec9babc910efb4314030c56dd798eaad12c426f7d180 \ - --hash=sha256:a4fcb0e2fcfd4bfe963837fad6d10943754fd79c1a50097d68540b6eb3d67f35 - # via google-cloud-storage -google-cloud-storage==3.1.0 \ - --hash=sha256:944273179897c7c8a07ee15f2e6466a02da0c7c4b9ecceac2a26017cb2972049 \ - --hash=sha256:eaf36966b68660a9633f03b067e4a10ce09f1377cae3ff9f2c699f69a81c66c6 - # via gcp-docuploader -google-crc32c==1.6.0 \ - --hash=sha256:05e2d8c9a2f853ff116db9706b4a27350587f341eda835f46db3c0a8c8ce2f24 \ - --hash=sha256:18e311c64008f1f1379158158bb3f0c8d72635b9eb4f9545f8cf990c5668e59d \ - --hash=sha256:236c87a46cdf06384f614e9092b82c05f81bd34b80248021f729396a78e55d7e \ - --hash=sha256:35834855408429cecf495cac67ccbab802de269e948e27478b1e47dfb6465e57 \ - --hash=sha256:386122eeaaa76951a8196310432c5b0ef3b53590ef4c317ec7588ec554fec5d2 \ - --hash=sha256:40b05ab32a5067525670880eb5d169529089a26fe35dce8891127aeddc1950e8 \ - --hash=sha256:48abd62ca76a2cbe034542ed1b6aee851b6f28aaca4e6551b5599b6f3ef175cc \ - --hash=sha256:50cf2a96da226dcbff8671233ecf37bf6e95de98b2a2ebadbfdf455e6d05df42 \ - --hash=sha256:51c4f54dd8c6dfeb58d1df5e4f7f97df8abf17a36626a217f169893d1d7f3e9f \ - --hash=sha256:5bcc90b34df28a4b38653c36bb5ada35671ad105c99cfe915fb5bed7ad6924aa \ - --hash=sha256:62f6d4a29fea082ac4a3c9be5e415218255cf11684ac6ef5488eea0c9132689b \ - --hash=sha256:6eceb6ad197656a1ff49ebfbbfa870678c75be4344feb35ac1edf694309413dc \ - --hash=sha256:7aec8e88a3583515f9e0957fe4f5f6d8d4997e36d0f61624e70469771584c760 \ - --hash=sha256:91ca8145b060679ec9176e6de4f89b07363d6805bd4760631ef254905503598d \ - --hash=sha256:a184243544811e4a50d345838a883733461e67578959ac59964e43cca2c791e7 \ - --hash=sha256:a9e4b426c3702f3cd23b933436487eb34e01e00327fac20c9aebb68ccf34117d \ - --hash=sha256:bb0966e1c50d0ef5bc743312cc730b533491d60585a9a08f897274e57c3f70e0 \ - --hash=sha256:bb8b3c75bd157010459b15222c3fd30577042a7060e29d42dabce449c087f2b3 \ - --hash=sha256:bd5e7d2445d1a958c266bfa5d04c39932dc54093fa391736dbfdb0f1929c1fb3 \ - --hash=sha256:c87d98c7c4a69066fd31701c4e10d178a648c2cac3452e62c6b24dc51f9fcc00 \ - --hash=sha256:d2952396dc604544ea7476b33fe87faedc24d666fb0c2d5ac971a2b9576ab871 \ - --hash=sha256:d8797406499f28b5ef791f339594b0b5fdedf54e203b5066675c406ba69d705c \ - --hash=sha256:d9e9913f7bd69e093b81da4535ce27af842e7bf371cde42d1ae9e9bd382dc0e9 \ - --hash=sha256:e2806553238cd076f0a55bddab37a532b53580e699ed8e5606d0de1f856b5205 \ - --hash=sha256:ebab974b1687509e5c973b5c4b8b146683e101e102e17a86bd196ecaa4d099fc \ - --hash=sha256:ed767bf4ba90104c1216b68111613f0d5926fb3780660ea1198fc469af410e9d \ - --hash=sha256:f7a1fc29803712f80879b0806cb83ab24ce62fc8daf0569f2204a0cfd7f68ed4 - # via - # google-cloud-storage - # google-resumable-media -google-resumable-media==2.7.2 \ - --hash=sha256:3ce7551e9fe6d99e9a126101d2536612bb73486721951e9562fee0f90c6ababa \ - --hash=sha256:5280aed4629f2b60b847b0d42f9857fd4935c11af266744df33d8074cae92fe0 - # via google-cloud-storage -googleapis-common-protos==1.69.0 \ - --hash=sha256:17835fdc4fa8da1d61cfe2d4d5d57becf7c61d4112f8d81c67eaa9d7ce43042d \ - --hash=sha256:5a46d58af72846f59009b9c4710425b9af2139555c71837081706b213b298187 - # via google-api-core -idna==3.10 \ - --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \ - --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3 - # via requests -nox==2025.2.9 \ - --hash=sha256:7d1e92d1918c6980d70aee9cf1c1d19d16faa71c4afe338fffd39e8a460e2067 \ - --hash=sha256:d50cd4ca568bd7621c2e6cbbc4845b3b7f7697f25d5fb0190ce8f4600be79768 - # via -r requirements.in -packaging==24.2 \ - --hash=sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759 \ - --hash=sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f - # via nox -platformdirs==4.3.6 \ - --hash=sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907 \ - --hash=sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb - # via virtualenv -proto-plus==1.26.0 \ - --hash=sha256:6e93d5f5ca267b54300880fff156b6a3386b3fa3f43b1da62e680fc0c586ef22 \ - --hash=sha256:bf2dfaa3da281fc3187d12d224c707cb57214fb2c22ba854eb0c105a3fb2d4d7 - # via google-api-core -protobuf==5.29.3 \ - --hash=sha256:0a18ed4a24198528f2333802eb075e59dea9d679ab7a6c5efb017a59004d849f \ - --hash=sha256:0eb32bfa5219fc8d4111803e9a690658aa2e6366384fd0851064b963b6d1f2a7 \ - --hash=sha256:3ea51771449e1035f26069c4c7fd51fba990d07bc55ba80701c78f886bf9c888 \ - --hash=sha256:5da0f41edaf117bde316404bad1a486cb4ededf8e4a54891296f648e8e076620 \ - --hash=sha256:6ce8cc3389a20693bfde6c6562e03474c40851b44975c9b2bf6df7d8c4f864da \ - --hash=sha256:84a57163a0ccef3f96e4b6a20516cedcf5bb3a95a657131c5c3ac62200d23252 \ - --hash=sha256:a4fa6f80816a9a0678429e84973f2f98cbc218cca434abe8db2ad0bffc98503a \ - --hash=sha256:a8434404bbf139aa9e1300dbf989667a83d42ddda9153d8ab76e0d5dcaca484e \ - --hash=sha256:b89c115d877892a512f79a8114564fb435943b59067615894c3b13cd3e1fa107 \ - --hash=sha256:c027e08a08be10b67c06bf2370b99c811c466398c357e615ca88c91c07f0910f \ - --hash=sha256:daaf63f70f25e8689c072cfad4334ca0ac1d1e05a92fc15c54eb9cf23c3efd84 - # via - # gcp-docuploader - # google-api-core - # googleapis-common-protos - # proto-plus -pyasn1==0.6.1 \ - --hash=sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629 \ - --hash=sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.4.1 \ - --hash=sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd \ - --hash=sha256:c28e2dbf9c06ad61c71a075c7e0f9fd0f1b0bb2d2ad4377f240d33ac2ab60a7c - # via google-auth -requests==2.32.3 \ - --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \ - --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 - # via - # google-api-core - # google-cloud-storage -rsa==4.9 \ - --hash=sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7 \ - --hash=sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21 - # via google-auth -six==1.17.0 \ - --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ - --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 - # via gcp-docuploader -tomli==2.2.1 \ - --hash=sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6 \ - --hash=sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd \ - --hash=sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c \ - --hash=sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b \ - --hash=sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8 \ - --hash=sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6 \ - --hash=sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77 \ - --hash=sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff \ - --hash=sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea \ - --hash=sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192 \ - --hash=sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249 \ - --hash=sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee \ - --hash=sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4 \ - --hash=sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98 \ - --hash=sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8 \ - --hash=sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4 \ - --hash=sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281 \ - --hash=sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744 \ - --hash=sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69 \ - --hash=sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13 \ - --hash=sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140 \ - --hash=sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e \ - --hash=sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e \ - --hash=sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc \ - --hash=sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff \ - --hash=sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec \ - --hash=sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2 \ - --hash=sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222 \ - --hash=sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106 \ - --hash=sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272 \ - --hash=sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a \ - --hash=sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7 - # via nox -urllib3==2.3.0 \ - --hash=sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df \ - --hash=sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d - # via requests -virtualenv==20.29.2 \ - --hash=sha256:fdaabebf6d03b5ba83ae0a02cfe96f48a716f4fae556461d180825866f75b728 \ - --hash=sha256:febddfc3d1ea571bdb1dc0f98d7b45d24def7428214d4fb73cc486c9568cce6a - # via nox diff --git a/packages/pandas-gbq/.kokoro/docs/common.cfg b/packages/pandas-gbq/.kokoro/docs/common.cfg deleted file mode 100644 index f30e60a480b3..000000000000 --- a/packages/pandas-gbq/.kokoro/docs/common.cfg +++ /dev/null @@ -1,67 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Build logs will be here -action { - define_artifacts { - regex: "**/*sponge_log.xml" - } -} - -# Download trampoline resources. -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" - -# Use the trampoline script to run in docker. -build_file: "python-bigquery-pandas/.kokoro/trampoline_v2.sh" - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/python-lib-docs" -} -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-bigquery-pandas/.kokoro/publish-docs.sh" -} - -env_vars: { - key: "STAGING_BUCKET" - value: "docs-staging" -} - -env_vars: { - key: "V2_STAGING_BUCKET" - # Push non-cloud library docs to `docs-staging-v2-dev` instead of the - # Cloud RAD bucket `docs-staging-v2` - value: "docs-staging-v2-dev" -} - -# It will upload the docker image after successful builds. -env_vars: { - key: "TRAMPOLINE_IMAGE_UPLOAD" - value: "true" -} - -# It will always build the docker image. -env_vars: { - key: "TRAMPOLINE_DOCKERFILE" - value: ".kokoro/docker/docs/Dockerfile" -} - -# Fetch the token needed for reporting release status to GitHub -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "yoshi-automation-github-key" - } - } -} - -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "docuploader_service_account" - } - } -} diff --git a/packages/pandas-gbq/.kokoro/docs/docs-presubmit.cfg b/packages/pandas-gbq/.kokoro/docs/docs-presubmit.cfg deleted file mode 100644 index 6c2a0fcf3195..000000000000 --- a/packages/pandas-gbq/.kokoro/docs/docs-presubmit.cfg +++ /dev/null @@ -1,28 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "STAGING_BUCKET" - value: "gcloud-python-test" -} - -env_vars: { - key: "V2_STAGING_BUCKET" - value: "gcloud-python-test" -} - -# We only upload the image in the main `docs` build. -env_vars: { - key: "TRAMPOLINE_IMAGE_UPLOAD" - value: "false" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-bigquery-pandas/.kokoro/build.sh" -} - -# Only run this nox session. -env_vars: { - key: "NOX_SESSION" - value: "docs docfx" -} diff --git a/packages/pandas-gbq/.kokoro/docs/docs.cfg b/packages/pandas-gbq/.kokoro/docs/docs.cfg deleted file mode 100644 index 8f43917d92fe..000000000000 --- a/packages/pandas-gbq/.kokoro/docs/docs.cfg +++ /dev/null @@ -1 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/publish-docs.sh b/packages/pandas-gbq/.kokoro/publish-docs.sh deleted file mode 100755 index 4ed4aaf1346f..000000000000 --- a/packages/pandas-gbq/.kokoro/publish-docs.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eo pipefail - -# Disable buffering, so that the logs stream through. -export PYTHONUNBUFFERED=1 - -export PATH="${HOME}/.local/bin:${PATH}" - -# build docs -nox -s docs - -# create metadata -python3.10 -m docuploader create-metadata \ - --name=$(jq --raw-output '.name // empty' .repo-metadata.json) \ - --version=$(python3.10 setup.py --version) \ - --language=$(jq --raw-output '.language // empty' .repo-metadata.json) \ - --distribution-name=$(python3.10 setup.py --name) \ - --product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \ - --github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \ - --issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json) - -cat docs.metadata - -# upload docs -python3.10 -m docuploader upload docs/_build/html --metadata-file docs.metadata --staging-bucket "${STAGING_BUCKET}" - - -# docfx yaml files -nox -s docfx - -# create metadata. -python3.10 -m docuploader create-metadata \ - --name=$(jq --raw-output '.name // empty' .repo-metadata.json) \ - --version=$(python3.10 setup.py --version) \ - --language=$(jq --raw-output '.language // empty' .repo-metadata.json) \ - --distribution-name=$(python3.10 setup.py --name) \ - --product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \ - --github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \ - --issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json) - -cat docs.metadata - -# upload docs -python3.10 -m docuploader upload docs/_build/html/docfx_yaml --metadata-file docs.metadata --destination-prefix docfx --staging-bucket "${V2_STAGING_BUCKET}" diff --git a/packages/pandas-gbq/.kokoro/release.sh b/packages/pandas-gbq/.kokoro/release.sh deleted file mode 100755 index 1757fe26f43b..000000000000 --- a/packages/pandas-gbq/.kokoro/release.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eo pipefail - -# Start the releasetool reporter -python3 -m pip install --require-hashes -r github/python-bigquery-pandas/.kokoro/requirements.txt -python3 -m releasetool publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script - -# Disable buffering, so that the logs stream through. -export PYTHONUNBUFFERED=1 - -# Move into the package, build the distribution and upload. -TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google-cloud-pypi-token-keystore-3") -cd github/python-bigquery-pandas -python3 setup.py sdist bdist_wheel -twine upload --username __token__ --password "${TWINE_PASSWORD}" dist/* diff --git a/packages/pandas-gbq/.kokoro/release/common.cfg b/packages/pandas-gbq/.kokoro/release/common.cfg deleted file mode 100644 index e8cb847b6ba7..000000000000 --- a/packages/pandas-gbq/.kokoro/release/common.cfg +++ /dev/null @@ -1,43 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Build logs will be here -action { - define_artifacts { - regex: "**/*sponge_log.xml" - } -} - -# Download trampoline resources. -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" - -# Use the trampoline script to run in docker. -build_file: "python-bigquery-pandas/.kokoro/trampoline.sh" - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/python-multi" -} -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-bigquery-pandas/.kokoro/release.sh" -} - -# Fetch PyPI password -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "google-cloud-pypi-token-keystore-3" - } - } -} - -# Store the packages we uploaded to PyPI. That way, we have a record of exactly -# what we published, which we can use to generate SBOMs and attestations. -action { - define_artifacts { - regex: "github/python-bigquery-pandas/**/*.tar.gz" - strip_prefix: "github/python-bigquery-pandas" - } -} diff --git a/packages/pandas-gbq/.kokoro/release/release.cfg b/packages/pandas-gbq/.kokoro/release/release.cfg deleted file mode 100644 index 8f43917d92fe..000000000000 --- a/packages/pandas-gbq/.kokoro/release/release.cfg +++ /dev/null @@ -1 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/requirements.in b/packages/pandas-gbq/.kokoro/requirements.in deleted file mode 100644 index fff4d9ce0d0a..000000000000 --- a/packages/pandas-gbq/.kokoro/requirements.in +++ /dev/null @@ -1,11 +0,0 @@ -gcp-docuploader -gcp-releasetool>=2 # required for compatibility with cryptography>=42.x -importlib-metadata -typing-extensions -twine -wheel -setuptools -nox>=2022.11.21 # required to remove dependency on py -charset-normalizer<3 -click<8.1.0 -cryptography>=42.0.5 diff --git a/packages/pandas-gbq/.kokoro/requirements.txt b/packages/pandas-gbq/.kokoro/requirements.txt deleted file mode 100644 index 6ad95a04a419..000000000000 --- a/packages/pandas-gbq/.kokoro/requirements.txt +++ /dev/null @@ -1,513 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.9 -# by the following command: -# -# pip-compile --allow-unsafe --generate-hashes requirements.in -# -argcomplete==3.5.1 \ - --hash=sha256:1a1d148bdaa3e3b93454900163403df41448a248af01b6e849edc5ac08e6c363 \ - --hash=sha256:eb1ee355aa2557bd3d0145de7b06b2a45b0ce461e1e7813f5d066039ab4177b4 - # via nox -attrs==24.2.0 \ - --hash=sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346 \ - --hash=sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2 - # via gcp-releasetool -backports-tarfile==1.2.0 \ - --hash=sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34 \ - --hash=sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991 - # via jaraco-context -cachetools==5.5.0 \ - --hash=sha256:02134e8439cdc2ffb62023ce1debca2944c3f289d66bb17ead3ab3dede74b292 \ - --hash=sha256:2cc24fb4cbe39633fb7badd9db9ca6295d766d9c2995f245725a46715d050f2a - # via google-auth -certifi==2024.8.30 \ - --hash=sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8 \ - --hash=sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9 - # via requests -cffi==1.17.1 \ - --hash=sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8 \ - --hash=sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2 \ - --hash=sha256:0e2b1fac190ae3ebfe37b979cc1ce69c81f4e4fe5746bb401dca63a9062cdaf1 \ - --hash=sha256:0f048dcf80db46f0098ccac01132761580d28e28bc0f78ae0d58048063317e15 \ - --hash=sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36 \ - --hash=sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824 \ - --hash=sha256:1d599671f396c4723d016dbddb72fe8e0397082b0a77a4fab8028923bec050e8 \ - --hash=sha256:28b16024becceed8c6dfbc75629e27788d8a3f9030691a1dbf9821a128b22c36 \ - --hash=sha256:2bb1a08b8008b281856e5971307cc386a8e9c5b625ac297e853d36da6efe9c17 \ - --hash=sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf \ - --hash=sha256:31000ec67d4221a71bd3f67df918b1f88f676f1c3b535a7eb473255fdc0b83fc \ - --hash=sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3 \ - --hash=sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed \ - --hash=sha256:45398b671ac6d70e67da8e4224a065cec6a93541bb7aebe1b198a61b58c7b702 \ - --hash=sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1 \ - --hash=sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8 \ - --hash=sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903 \ - --hash=sha256:5da5719280082ac6bd9aa7becb3938dc9f9cbd57fac7d2871717b1feb0902ab6 \ - --hash=sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d \ - --hash=sha256:636062ea65bd0195bc012fea9321aca499c0504409f413dc88af450b57ffd03b \ - --hash=sha256:6883e737d7d9e4899a8a695e00ec36bd4e5e4f18fabe0aca0efe0a4b44cdb13e \ - --hash=sha256:6b8b4a92e1c65048ff98cfe1f735ef8f1ceb72e3d5f0c25fdb12087a23da22be \ - --hash=sha256:6f17be4345073b0a7b8ea599688f692ac3ef23ce28e5df79c04de519dbc4912c \ - --hash=sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683 \ - --hash=sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9 \ - --hash=sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c \ - --hash=sha256:7596d6620d3fa590f677e9ee430df2958d2d6d6de2feeae5b20e82c00b76fbf8 \ - --hash=sha256:78122be759c3f8a014ce010908ae03364d00a1f81ab5c7f4a7a5120607ea56e1 \ - --hash=sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4 \ - --hash=sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655 \ - --hash=sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67 \ - --hash=sha256:9755e4345d1ec879e3849e62222a18c7174d65a6a92d5b346b1863912168b595 \ - --hash=sha256:98e3969bcff97cae1b2def8ba499ea3d6f31ddfdb7635374834cf89a1a08ecf0 \ - --hash=sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65 \ - --hash=sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41 \ - --hash=sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6 \ - --hash=sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401 \ - --hash=sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6 \ - --hash=sha256:ad9413ccdeda48c5afdae7e4fa2192157e991ff761e7ab8fdd8926f40b160cc3 \ - --hash=sha256:b2ab587605f4ba0bf81dc0cb08a41bd1c0a5906bd59243d56bad7668a6fc6c16 \ - --hash=sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93 \ - --hash=sha256:c03e868a0b3bc35839ba98e74211ed2b05d2119be4e8a0f224fba9384f1fe02e \ - --hash=sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4 \ - --hash=sha256:c7eac2ef9b63c79431bc4b25f1cd649d7f061a28808cbc6c47b534bd789ef964 \ - --hash=sha256:c9c3d058ebabb74db66e431095118094d06abf53284d9c81f27300d0e0d8bc7c \ - --hash=sha256:ca74b8dbe6e8e8263c0ffd60277de77dcee6c837a3d0881d8c1ead7268c9e576 \ - --hash=sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0 \ - --hash=sha256:cdf5ce3acdfd1661132f2a9c19cac174758dc2352bfe37d98aa7512c6b7178b3 \ - --hash=sha256:d016c76bdd850f3c626af19b0542c9677ba156e4ee4fccfdd7848803533ef662 \ - --hash=sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3 \ - --hash=sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff \ - --hash=sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5 \ - --hash=sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd \ - --hash=sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f \ - --hash=sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5 \ - --hash=sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14 \ - --hash=sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d \ - --hash=sha256:e221cf152cff04059d011ee126477f0d9588303eb57e88923578ace7baad17f9 \ - --hash=sha256:e31ae45bc2e29f6b2abd0de1cc3b9d5205aa847cafaecb8af1476a609a2f6eb7 \ - --hash=sha256:edae79245293e15384b51f88b00613ba9f7198016a5948b5dddf4917d4d26382 \ - --hash=sha256:f1e22e8c4419538cb197e4dd60acc919d7696e5ef98ee4da4e01d3f8cfa4cc5a \ - --hash=sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e \ - --hash=sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a \ - --hash=sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4 \ - --hash=sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99 \ - --hash=sha256:f7f5baafcc48261359e14bcd6d9bff6d4b28d9103847c9e136694cb0501aef87 \ - --hash=sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b - # via cryptography -charset-normalizer==2.1.1 \ - --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 \ - --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f - # via - # -r requirements.in - # requests -click==8.0.4 \ - --hash=sha256:6a7a62563bbfabfda3a38f3023a1db4a35978c0abd76f6c9605ecd6554d6d9b1 \ - --hash=sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb - # via - # -r requirements.in - # gcp-docuploader - # gcp-releasetool -colorlog==6.8.2 \ - --hash=sha256:3e3e079a41feb5a1b64f978b5ea4f46040a94f11f0e8bbb8261e3dbbeca64d44 \ - --hash=sha256:4dcbb62368e2800cb3c5abd348da7e53f6c362dda502ec27c560b2e58a66bd33 - # via - # gcp-docuploader - # nox -cryptography==44.0.1 \ - --hash=sha256:00918d859aa4e57db8299607086f793fa7813ae2ff5a4637e318a25ef82730f7 \ - --hash=sha256:1e8d181e90a777b63f3f0caa836844a1182f1f265687fac2115fcf245f5fbec3 \ - --hash=sha256:1f9a92144fa0c877117e9748c74501bea842f93d21ee00b0cf922846d9d0b183 \ - --hash=sha256:21377472ca4ada2906bc313168c9dc7b1d7ca417b63c1c3011d0c74b7de9ae69 \ - --hash=sha256:24979e9f2040c953a94bf3c6782e67795a4c260734e5264dceea65c8f4bae64a \ - --hash=sha256:2a46a89ad3e6176223b632056f321bc7de36b9f9b93b2cc1cccf935a3849dc62 \ - --hash=sha256:322eb03ecc62784536bc173f1483e76747aafeb69c8728df48537eb431cd1911 \ - --hash=sha256:436df4f203482f41aad60ed1813811ac4ab102765ecae7a2bbb1dbb66dcff5a7 \ - --hash=sha256:4f422e8c6a28cf8b7f883eb790695d6d45b0c385a2583073f3cec434cc705e1a \ - --hash=sha256:53f23339864b617a3dfc2b0ac8d5c432625c80014c25caac9082314e9de56f41 \ - --hash=sha256:5fed5cd6102bb4eb843e3315d2bf25fede494509bddadb81e03a859c1bc17b83 \ - --hash=sha256:610a83540765a8d8ce0f351ce42e26e53e1f774a6efb71eb1b41eb01d01c3d12 \ - --hash=sha256:6c8acf6f3d1f47acb2248ec3ea261171a671f3d9428e34ad0357148d492c7864 \ - --hash=sha256:6f76fdd6fd048576a04c5210d53aa04ca34d2ed63336d4abd306d0cbe298fddf \ - --hash=sha256:72198e2b5925155497a5a3e8c216c7fb3e64c16ccee11f0e7da272fa93b35c4c \ - --hash=sha256:887143b9ff6bad2b7570da75a7fe8bbf5f65276365ac259a5d2d5147a73775f2 \ - --hash=sha256:888fcc3fce0c888785a4876ca55f9f43787f4c5c1cc1e2e0da71ad481ff82c5b \ - --hash=sha256:8e6a85a93d0642bd774460a86513c5d9d80b5c002ca9693e63f6e540f1815ed0 \ - --hash=sha256:94f99f2b943b354a5b6307d7e8d19f5c423a794462bde2bf310c770ba052b1c4 \ - --hash=sha256:9b336599e2cb77b1008cb2ac264b290803ec5e8e89d618a5e978ff5eb6f715d9 \ - --hash=sha256:a2d8a7045e1ab9b9f803f0d9531ead85f90c5f2859e653b61497228b18452008 \ - --hash=sha256:b8272f257cf1cbd3f2e120f14c68bff2b6bdfcc157fafdee84a1b795efd72862 \ - --hash=sha256:bf688f615c29bfe9dfc44312ca470989279f0e94bb9f631f85e3459af8efc009 \ - --hash=sha256:d9c5b9f698a83c8bd71e0f4d3f9f839ef244798e5ffe96febfa9714717db7af7 \ - --hash=sha256:dd7c7e2d71d908dc0f8d2027e1604102140d84b155e658c20e8ad1304317691f \ - --hash=sha256:df978682c1504fc93b3209de21aeabf2375cb1571d4e61907b3e7a2540e83026 \ - --hash=sha256:e403f7f766ded778ecdb790da786b418a9f2394f36e8cc8b796cc056ab05f44f \ - --hash=sha256:eb3889330f2a4a148abead555399ec9a32b13b7c8ba969b72d8e500eb7ef84cd \ - --hash=sha256:f4daefc971c2d1f82f03097dc6f216744a6cd2ac0f04c68fb935ea2ba2a0d420 \ - --hash=sha256:f51f5705ab27898afda1aaa430f34ad90dc117421057782022edf0600bec5f14 \ - --hash=sha256:fd0ee90072861e276b0ff08bd627abec29e32a53b2be44e41dbcdf87cbee2b00 - # via - # -r requirements.in - # gcp-releasetool - # secretstorage -distlib==0.3.9 \ - --hash=sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87 \ - --hash=sha256:a60f20dea646b8a33f3e7772f74dc0b2d0772d2837ee1342a00645c81edf9403 - # via virtualenv -docutils==0.21.2 \ - --hash=sha256:3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f \ - --hash=sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2 - # via readme-renderer -filelock==3.16.1 \ - --hash=sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0 \ - --hash=sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435 - # via virtualenv -gcp-docuploader==0.6.5 \ - --hash=sha256:30221d4ac3e5a2b9c69aa52fdbef68cc3f27d0e6d0d90e220fc024584b8d2318 \ - --hash=sha256:b7458ef93f605b9d46a4bf3a8dc1755dad1f31d030c8679edf304e343b347eea - # via -r requirements.in -gcp-releasetool==2.1.1 \ - --hash=sha256:25639269f4eae510094f9dbed9894977e1966933211eb155a451deebc3fc0b30 \ - --hash=sha256:845f4ded3d9bfe8cc7fdaad789e83f4ea014affa77785259a7ddac4b243e099e - # via -r requirements.in -google-api-core==2.21.0 \ - --hash=sha256:4a152fd11a9f774ea606388d423b68aa7e6d6a0ffe4c8266f74979613ec09f81 \ - --hash=sha256:6869eacb2a37720380ba5898312af79a4d30b8bca1548fb4093e0697dc4bdf5d - # via - # google-cloud-core - # google-cloud-storage -google-auth==2.35.0 \ - --hash=sha256:25df55f327ef021de8be50bad0dfd4a916ad0de96da86cd05661c9297723ad3f \ - --hash=sha256:f4c64ed4e01e8e8b646ef34c018f8bf3338df0c8e37d8b3bba40e7f574a3278a - # via - # gcp-releasetool - # google-api-core - # google-cloud-core - # google-cloud-storage -google-cloud-core==2.4.1 \ - --hash=sha256:9b7749272a812bde58fff28868d0c5e2f585b82f37e09a1f6ed2d4d10f134073 \ - --hash=sha256:a9e6a4422b9ac5c29f79a0ede9485473338e2ce78d91f2370c01e730eab22e61 - # via google-cloud-storage -google-cloud-storage==2.18.2 \ - --hash=sha256:97a4d45c368b7d401ed48c4fdfe86e1e1cb96401c9e199e419d289e2c0370166 \ - --hash=sha256:aaf7acd70cdad9f274d29332673fcab98708d0e1f4dceb5a5356aaef06af4d99 - # via gcp-docuploader -google-crc32c==1.6.0 \ - --hash=sha256:05e2d8c9a2f853ff116db9706b4a27350587f341eda835f46db3c0a8c8ce2f24 \ - --hash=sha256:18e311c64008f1f1379158158bb3f0c8d72635b9eb4f9545f8cf990c5668e59d \ - --hash=sha256:236c87a46cdf06384f614e9092b82c05f81bd34b80248021f729396a78e55d7e \ - --hash=sha256:35834855408429cecf495cac67ccbab802de269e948e27478b1e47dfb6465e57 \ - --hash=sha256:386122eeaaa76951a8196310432c5b0ef3b53590ef4c317ec7588ec554fec5d2 \ - --hash=sha256:40b05ab32a5067525670880eb5d169529089a26fe35dce8891127aeddc1950e8 \ - --hash=sha256:48abd62ca76a2cbe034542ed1b6aee851b6f28aaca4e6551b5599b6f3ef175cc \ - --hash=sha256:50cf2a96da226dcbff8671233ecf37bf6e95de98b2a2ebadbfdf455e6d05df42 \ - --hash=sha256:51c4f54dd8c6dfeb58d1df5e4f7f97df8abf17a36626a217f169893d1d7f3e9f \ - --hash=sha256:5bcc90b34df28a4b38653c36bb5ada35671ad105c99cfe915fb5bed7ad6924aa \ - --hash=sha256:62f6d4a29fea082ac4a3c9be5e415218255cf11684ac6ef5488eea0c9132689b \ - --hash=sha256:6eceb6ad197656a1ff49ebfbbfa870678c75be4344feb35ac1edf694309413dc \ - --hash=sha256:7aec8e88a3583515f9e0957fe4f5f6d8d4997e36d0f61624e70469771584c760 \ - --hash=sha256:91ca8145b060679ec9176e6de4f89b07363d6805bd4760631ef254905503598d \ - --hash=sha256:a184243544811e4a50d345838a883733461e67578959ac59964e43cca2c791e7 \ - --hash=sha256:a9e4b426c3702f3cd23b933436487eb34e01e00327fac20c9aebb68ccf34117d \ - --hash=sha256:bb0966e1c50d0ef5bc743312cc730b533491d60585a9a08f897274e57c3f70e0 \ - --hash=sha256:bb8b3c75bd157010459b15222c3fd30577042a7060e29d42dabce449c087f2b3 \ - --hash=sha256:bd5e7d2445d1a958c266bfa5d04c39932dc54093fa391736dbfdb0f1929c1fb3 \ - --hash=sha256:c87d98c7c4a69066fd31701c4e10d178a648c2cac3452e62c6b24dc51f9fcc00 \ - --hash=sha256:d2952396dc604544ea7476b33fe87faedc24d666fb0c2d5ac971a2b9576ab871 \ - --hash=sha256:d8797406499f28b5ef791f339594b0b5fdedf54e203b5066675c406ba69d705c \ - --hash=sha256:d9e9913f7bd69e093b81da4535ce27af842e7bf371cde42d1ae9e9bd382dc0e9 \ - --hash=sha256:e2806553238cd076f0a55bddab37a532b53580e699ed8e5606d0de1f856b5205 \ - --hash=sha256:ebab974b1687509e5c973b5c4b8b146683e101e102e17a86bd196ecaa4d099fc \ - --hash=sha256:ed767bf4ba90104c1216b68111613f0d5926fb3780660ea1198fc469af410e9d \ - --hash=sha256:f7a1fc29803712f80879b0806cb83ab24ce62fc8daf0569f2204a0cfd7f68ed4 - # via - # google-cloud-storage - # google-resumable-media -google-resumable-media==2.7.2 \ - --hash=sha256:3ce7551e9fe6d99e9a126101d2536612bb73486721951e9562fee0f90c6ababa \ - --hash=sha256:5280aed4629f2b60b847b0d42f9857fd4935c11af266744df33d8074cae92fe0 - # via google-cloud-storage -googleapis-common-protos==1.65.0 \ - --hash=sha256:2972e6c496f435b92590fd54045060867f3fe9be2c82ab148fc8885035479a63 \ - --hash=sha256:334a29d07cddc3aa01dee4988f9afd9b2916ee2ff49d6b757155dc0d197852c0 - # via google-api-core -idna==3.10 \ - --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \ - --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3 - # via requests -importlib-metadata==8.5.0 \ - --hash=sha256:45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b \ - --hash=sha256:71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7 - # via - # -r requirements.in - # keyring - # twine -jaraco-classes==3.4.0 \ - --hash=sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd \ - --hash=sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790 - # via keyring -jaraco-context==6.0.1 \ - --hash=sha256:9bae4ea555cf0b14938dc0aee7c9f32ed303aa20a3b73e7dc80111628792d1b3 \ - --hash=sha256:f797fc481b490edb305122c9181830a3a5b76d84ef6d1aef2fb9b47ab956f9e4 - # via keyring -jaraco-functools==4.1.0 \ - --hash=sha256:70f7e0e2ae076498e212562325e805204fc092d7b4c17e0e86c959e249701a9d \ - --hash=sha256:ad159f13428bc4acbf5541ad6dec511f91573b90fba04df61dafa2a1231cf649 - # via keyring -jeepney==0.8.0 \ - --hash=sha256:5efe48d255973902f6badc3ce55e2aa6c5c3b3bc642059ef3a91247bcfcc5806 \ - --hash=sha256:c0a454ad016ca575060802ee4d590dd912e35c122fa04e70306de3d076cce755 - # via - # keyring - # secretstorage -jinja2==3.1.5 \ - --hash=sha256:8fefff8dc3034e27bb80d67c671eb8a9bc424c0ef4c0826edbff304cceff43bb \ - --hash=sha256:aba0f4dc9ed8013c424088f68a5c226f7d6097ed89b246d7749c2ec4175c6adb - # via gcp-releasetool -keyring==25.4.1 \ - --hash=sha256:5426f817cf7f6f007ba5ec722b1bcad95a75b27d780343772ad76b17cb47b0bf \ - --hash=sha256:b07ebc55f3e8ed86ac81dd31ef14e81ace9dd9c3d4b5d77a6e9a2016d0d71a1b - # via - # gcp-releasetool - # twine -markdown-it-py==3.0.0 \ - --hash=sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1 \ - --hash=sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb - # via rich -markupsafe==3.0.1 \ - --hash=sha256:0778de17cff1acaeccc3ff30cd99a3fd5c50fc58ad3d6c0e0c4c58092b859396 \ - --hash=sha256:0f84af7e813784feb4d5e4ff7db633aba6c8ca64a833f61d8e4eade234ef0c38 \ - --hash=sha256:17b2aea42a7280db02ac644db1d634ad47dcc96faf38ab304fe26ba2680d359a \ - --hash=sha256:242d6860f1fd9191aef5fae22b51c5c19767f93fb9ead4d21924e0bcb17619d8 \ - --hash=sha256:244dbe463d5fb6d7ce161301a03a6fe744dac9072328ba9fc82289238582697b \ - --hash=sha256:26627785a54a947f6d7336ce5963569b5d75614619e75193bdb4e06e21d447ad \ - --hash=sha256:2a4b34a8d14649315c4bc26bbfa352663eb51d146e35eef231dd739d54a5430a \ - --hash=sha256:2ae99f31f47d849758a687102afdd05bd3d3ff7dbab0a8f1587981b58a76152a \ - --hash=sha256:312387403cd40699ab91d50735ea7a507b788091c416dd007eac54434aee51da \ - --hash=sha256:3341c043c37d78cc5ae6e3e305e988532b072329639007fd408a476642a89fd6 \ - --hash=sha256:33d1c36b90e570ba7785dacd1faaf091203d9942bc036118fab8110a401eb1a8 \ - --hash=sha256:3e683ee4f5d0fa2dde4db77ed8dd8a876686e3fc417655c2ece9a90576905344 \ - --hash=sha256:3ffb4a8e7d46ed96ae48805746755fadd0909fea2306f93d5d8233ba23dda12a \ - --hash=sha256:40621d60d0e58aa573b68ac5e2d6b20d44392878e0bfc159012a5787c4e35bc8 \ - --hash=sha256:40f1e10d51c92859765522cbd79c5c8989f40f0419614bcdc5015e7b6bf97fc5 \ - --hash=sha256:45d42d132cff577c92bfba536aefcfea7e26efb975bd455db4e6602f5c9f45e7 \ - --hash=sha256:48488d999ed50ba8d38c581d67e496f955821dc183883550a6fbc7f1aefdc170 \ - --hash=sha256:4935dd7883f1d50e2ffecca0aa33dc1946a94c8f3fdafb8df5c330e48f71b132 \ - --hash=sha256:4c2d64fdba74ad16138300815cfdc6ab2f4647e23ced81f59e940d7d4a1469d9 \ - --hash=sha256:4c8817557d0de9349109acb38b9dd570b03cc5014e8aabf1cbddc6e81005becd \ - --hash=sha256:4ffaaac913c3f7345579db4f33b0020db693f302ca5137f106060316761beea9 \ - --hash=sha256:5a4cb365cb49b750bdb60b846b0c0bc49ed62e59a76635095a179d440540c346 \ - --hash=sha256:62fada2c942702ef8952754abfc1a9f7658a4d5460fabe95ac7ec2cbe0d02abc \ - --hash=sha256:67c519635a4f64e495c50e3107d9b4075aec33634272b5db1cde839e07367589 \ - --hash=sha256:6a54c43d3ec4cf2a39f4387ad044221c66a376e58c0d0e971d47c475ba79c6b5 \ - --hash=sha256:7044312a928a66a4c2a22644147bc61a199c1709712069a344a3fb5cfcf16915 \ - --hash=sha256:730d86af59e0e43ce277bb83970530dd223bf7f2a838e086b50affa6ec5f9295 \ - --hash=sha256:800100d45176652ded796134277ecb13640c1a537cad3b8b53da45aa96330453 \ - --hash=sha256:80fcbf3add8790caddfab6764bde258b5d09aefbe9169c183f88a7410f0f6dea \ - --hash=sha256:82b5dba6eb1bcc29cc305a18a3c5365d2af06ee71b123216416f7e20d2a84e5b \ - --hash=sha256:852dc840f6d7c985603e60b5deaae1d89c56cb038b577f6b5b8c808c97580f1d \ - --hash=sha256:8ad4ad1429cd4f315f32ef263c1342166695fad76c100c5d979c45d5570ed58b \ - --hash=sha256:8ae369e84466aa70f3154ee23c1451fda10a8ee1b63923ce76667e3077f2b0c4 \ - --hash=sha256:93e8248d650e7e9d49e8251f883eed60ecbc0e8ffd6349e18550925e31bd029b \ - --hash=sha256:973a371a55ce9ed333a3a0f8e0bcfae9e0d637711534bcb11e130af2ab9334e7 \ - --hash=sha256:9ba25a71ebf05b9bb0e2ae99f8bc08a07ee8e98c612175087112656ca0f5c8bf \ - --hash=sha256:a10860e00ded1dd0a65b83e717af28845bb7bd16d8ace40fe5531491de76b79f \ - --hash=sha256:a4792d3b3a6dfafefdf8e937f14906a51bd27025a36f4b188728a73382231d91 \ - --hash=sha256:a7420ceda262dbb4b8d839a4ec63d61c261e4e77677ed7c66c99f4e7cb5030dd \ - --hash=sha256:ad91738f14eb8da0ff82f2acd0098b6257621410dcbd4df20aaa5b4233d75a50 \ - --hash=sha256:b6a387d61fe41cdf7ea95b38e9af11cfb1a63499af2759444b99185c4ab33f5b \ - --hash=sha256:b954093679d5750495725ea6f88409946d69cfb25ea7b4c846eef5044194f583 \ - --hash=sha256:bbde71a705f8e9e4c3e9e33db69341d040c827c7afa6789b14c6e16776074f5a \ - --hash=sha256:beeebf760a9c1f4c07ef6a53465e8cfa776ea6a2021eda0d0417ec41043fe984 \ - --hash=sha256:c91b394f7601438ff79a4b93d16be92f216adb57d813a78be4446fe0f6bc2d8c \ - --hash=sha256:c97ff7fedf56d86bae92fa0a646ce1a0ec7509a7578e1ed238731ba13aabcd1c \ - --hash=sha256:cb53e2a99df28eee3b5f4fea166020d3ef9116fdc5764bc5117486e6d1211b25 \ - --hash=sha256:cbf445eb5628981a80f54087f9acdbf84f9b7d862756110d172993b9a5ae81aa \ - --hash=sha256:d06b24c686a34c86c8c1fba923181eae6b10565e4d80bdd7bc1c8e2f11247aa4 \ - --hash=sha256:d98e66a24497637dd31ccab090b34392dddb1f2f811c4b4cd80c230205c074a3 \ - --hash=sha256:db15ce28e1e127a0013dfb8ac243a8e392db8c61eae113337536edb28bdc1f97 \ - --hash=sha256:db842712984e91707437461930e6011e60b39136c7331e971952bb30465bc1a1 \ - --hash=sha256:e24bfe89c6ac4c31792793ad9f861b8f6dc4546ac6dc8f1c9083c7c4f2b335cd \ - --hash=sha256:e81c52638315ff4ac1b533d427f50bc0afc746deb949210bc85f05d4f15fd772 \ - --hash=sha256:e9393357f19954248b00bed7c56f29a25c930593a77630c719653d51e7669c2a \ - --hash=sha256:ee3941769bd2522fe39222206f6dd97ae83c442a94c90f2b7a25d847d40f4729 \ - --hash=sha256:f31ae06f1328595d762c9a2bf29dafd8621c7d3adc130cbb46278079758779ca \ - --hash=sha256:f94190df587738280d544971500b9cafc9b950d32efcb1fba9ac10d84e6aa4e6 \ - --hash=sha256:fa7d686ed9883f3d664d39d5a8e74d3c5f63e603c2e3ff0abcba23eac6542635 \ - --hash=sha256:fb532dd9900381d2e8f48172ddc5a59db4c445a11b9fab40b3b786da40d3b56b \ - --hash=sha256:fe32482b37b4b00c7a52a07211b479653b7fe4f22b2e481b9a9b099d8a430f2f - # via jinja2 -mdurl==0.1.2 \ - --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ - --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba - # via markdown-it-py -more-itertools==10.5.0 \ - --hash=sha256:037b0d3203ce90cca8ab1defbbdac29d5f993fc20131f3664dc8d6acfa872aef \ - --hash=sha256:5482bfef7849c25dc3c6dd53a6173ae4795da2a41a80faea6700d9f5846c5da6 - # via - # jaraco-classes - # jaraco-functools -nh3==0.2.18 \ - --hash=sha256:0411beb0589eacb6734f28d5497ca2ed379eafab8ad8c84b31bb5c34072b7164 \ - --hash=sha256:14c5a72e9fe82aea5fe3072116ad4661af5cf8e8ff8fc5ad3450f123e4925e86 \ - --hash=sha256:19aaba96e0f795bd0a6c56291495ff59364f4300d4a39b29a0abc9cb3774a84b \ - --hash=sha256:34c03fa78e328c691f982b7c03d4423bdfd7da69cd707fe572f544cf74ac23ad \ - --hash=sha256:36c95d4b70530b320b365659bb5034341316e6a9b30f0b25fa9c9eff4c27a204 \ - --hash=sha256:3a157ab149e591bb638a55c8c6bcb8cdb559c8b12c13a8affaba6cedfe51713a \ - --hash=sha256:42c64511469005058cd17cc1537578eac40ae9f7200bedcfd1fc1a05f4f8c200 \ - --hash=sha256:5f36b271dae35c465ef5e9090e1fdaba4a60a56f0bb0ba03e0932a66f28b9189 \ - --hash=sha256:6955369e4d9f48f41e3f238a9e60f9410645db7e07435e62c6a9ea6135a4907f \ - --hash=sha256:7b7c2a3c9eb1a827d42539aa64091640bd275b81e097cd1d8d82ef91ffa2e811 \ - --hash=sha256:8ce0f819d2f1933953fca255db2471ad58184a60508f03e6285e5114b6254844 \ - --hash=sha256:94a166927e53972a9698af9542ace4e38b9de50c34352b962f4d9a7d4c927af4 \ - --hash=sha256:a7f1b5b2c15866f2db413a3649a8fe4fd7b428ae58be2c0f6bca5eefd53ca2be \ - --hash=sha256:c8b3a1cebcba9b3669ed1a84cc65bf005728d2f0bc1ed2a6594a992e817f3a50 \ - --hash=sha256:de3ceed6e661954871d6cd78b410213bdcb136f79aafe22aa7182e028b8c7307 \ - --hash=sha256:f0eca9ca8628dbb4e916ae2491d72957fdd35f7a5d326b7032a345f111ac07fe - # via readme-renderer -nox==2024.10.9 \ - --hash=sha256:1d36f309a0a2a853e9bccb76bbef6bb118ba92fa92674d15604ca99adeb29eab \ - --hash=sha256:7aa9dc8d1c27e9f45ab046ffd1c3b2c4f7c91755304769df231308849ebded95 - # via -r requirements.in -packaging==24.1 \ - --hash=sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002 \ - --hash=sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124 - # via - # gcp-releasetool - # nox -pkginfo==1.10.0 \ - --hash=sha256:5df73835398d10db79f8eecd5cd86b1f6d29317589ea70796994d49399af6297 \ - --hash=sha256:889a6da2ed7ffc58ab5b900d888ddce90bce912f2d2de1dc1c26f4cb9fe65097 - # via twine -platformdirs==4.3.6 \ - --hash=sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907 \ - --hash=sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb - # via virtualenv -proto-plus==1.24.0 \ - --hash=sha256:30b72a5ecafe4406b0d339db35b56c4059064e69227b8c3bda7462397f966445 \ - --hash=sha256:402576830425e5f6ce4c2a6702400ac79897dab0b4343821aa5188b0fab81a12 - # via google-api-core -protobuf==5.28.2 \ - --hash=sha256:2c69461a7fcc8e24be697624c09a839976d82ae75062b11a0972e41fd2cd9132 \ - --hash=sha256:35cfcb15f213449af7ff6198d6eb5f739c37d7e4f1c09b5d0641babf2cc0c68f \ - --hash=sha256:52235802093bd8a2811abbe8bf0ab9c5f54cca0a751fdd3f6ac2a21438bffece \ - --hash=sha256:59379674ff119717404f7454647913787034f03fe7049cbef1d74a97bb4593f0 \ - --hash=sha256:5e8a95246d581eef20471b5d5ba010d55f66740942b95ba9b872d918c459452f \ - --hash=sha256:87317e9bcda04a32f2ee82089a204d3a2f0d3c8aeed16568c7daf4756e4f1fe0 \ - --hash=sha256:8ddc60bf374785fb7cb12510b267f59067fa10087325b8e1855b898a0d81d276 \ - --hash=sha256:a8b9403fc70764b08d2f593ce44f1d2920c5077bf7d311fefec999f8c40f78b7 \ - --hash=sha256:c0ea0123dac3399a2eeb1a1443d82b7afc9ff40241433296769f7da42d142ec3 \ - --hash=sha256:ca53faf29896c526863366a52a8f4d88e69cd04ec9571ed6082fa117fac3ab36 \ - --hash=sha256:eeea10f3dc0ac7e6b4933d32db20662902b4ab81bf28df12218aa389e9c2102d - # via - # gcp-docuploader - # gcp-releasetool - # google-api-core - # googleapis-common-protos - # proto-plus -pyasn1==0.6.1 \ - --hash=sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629 \ - --hash=sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.4.1 \ - --hash=sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd \ - --hash=sha256:c28e2dbf9c06ad61c71a075c7e0f9fd0f1b0bb2d2ad4377f240d33ac2ab60a7c - # via google-auth -pycparser==2.22 \ - --hash=sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6 \ - --hash=sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc - # via cffi -pygments==2.18.0 \ - --hash=sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199 \ - --hash=sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a - # via - # readme-renderer - # rich -pyjwt==2.9.0 \ - --hash=sha256:3b02fb0f44517787776cf48f2ae25d8e14f300e6d7545a4315cee571a415e850 \ - --hash=sha256:7e1e5b56cc735432a7369cbfa0efe50fa113ebecdc04ae6922deba8b84582d0c - # via gcp-releasetool -pyperclip==1.9.0 \ - --hash=sha256:b7de0142ddc81bfc5c7507eea19da920b92252b548b96186caf94a5e2527d310 - # via gcp-releasetool -python-dateutil==2.9.0.post0 \ - --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ - --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 - # via gcp-releasetool -readme-renderer==44.0 \ - --hash=sha256:2fbca89b81a08526aadf1357a8c2ae889ec05fb03f5da67f9769c9a592166151 \ - --hash=sha256:8712034eabbfa6805cacf1402b4eeb2a73028f72d1166d6f5cb7f9c047c5d1e1 - # via twine -requests==2.32.3 \ - --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \ - --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 - # via - # gcp-releasetool - # google-api-core - # google-cloud-storage - # requests-toolbelt - # twine -requests-toolbelt==1.0.0 \ - --hash=sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6 \ - --hash=sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06 - # via twine -rfc3986==2.0.0 \ - --hash=sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd \ - --hash=sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c - # via twine -rich==13.9.2 \ - --hash=sha256:51a2c62057461aaf7152b4d611168f93a9fc73068f8ded2790f29fe2b5366d0c \ - --hash=sha256:8c82a3d3f8dcfe9e734771313e606b39d8247bb6b826e196f4914b333b743cf1 - # via twine -rsa==4.9 \ - --hash=sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7 \ - --hash=sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21 - # via google-auth -secretstorage==3.3.3 \ - --hash=sha256:2403533ef369eca6d2ba81718576c5e0f564d5cca1b58f73a8b23e7d4eeebd77 \ - --hash=sha256:f356e6628222568e3af06f2eba8df495efa13b3b63081dafd4f7d9a7b7bc9f99 - # via keyring -six==1.16.0 \ - --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ - --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 - # via - # gcp-docuploader - # python-dateutil -tomli==2.0.2 \ - --hash=sha256:2ebe24485c53d303f690b0ec092806a085f07af5a5aa1464f3931eec36caaa38 \ - --hash=sha256:d46d457a85337051c36524bc5349dd91b1877838e2979ac5ced3e710ed8a60ed - # via nox -twine==5.1.1 \ - --hash=sha256:215dbe7b4b94c2c50a7315c0275d2258399280fbb7d04182c7e55e24b5f93997 \ - --hash=sha256:9aa0825139c02b3434d913545c7b847a21c835e11597f5255842d457da2322db - # via -r requirements.in -typing-extensions==4.12.2 \ - --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \ - --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8 - # via - # -r requirements.in - # rich -urllib3==2.2.3 \ - --hash=sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac \ - --hash=sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9 - # via - # requests - # twine -virtualenv==20.26.6 \ - --hash=sha256:280aede09a2a5c317e409a00102e7077c6432c5a38f0ef938e643805a7ad2c48 \ - --hash=sha256:7345cc5b25405607a624d8418154577459c3e0277f5466dd79c49d5e492995f2 - # via nox -wheel==0.44.0 \ - --hash=sha256:2376a90c98cc337d18623527a97c31797bd02bad0033d41547043a1cbfbe448f \ - --hash=sha256:a29c3f2817e95ab89aa4660681ad547c0e9547f20e75b0562fe7723c9a2a9d49 - # via -r requirements.in -zipp==3.20.2 \ - --hash=sha256:a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350 \ - --hash=sha256:bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29 - # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -setuptools==75.1.0 \ - --hash=sha256:35ab7fd3bcd95e6b7fd704e4a1539513edad446c097797f2985e0e4b960772f2 \ - --hash=sha256:d59a21b17a275fb872a9c3dae73963160ae079f1049ed956880cd7c09b120538 - # via -r requirements.in From 4af9d92b257e45c093764793fd91919815963786 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 6 Mar 2025 15:50:20 -0500 Subject: [PATCH 477/519] fix: resolve issue where pre-release versions of dependencies are installed (#890) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: resolve issue where pre-release versions of dependencies are installed * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * Delete .kokoro/docker/docs/requirements.txt * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * See https://github.com/googleapis/python-bigquery-pandas/pull/889/files --------- Co-authored-by: Owl Bot --- packages/pandas-gbq/setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index 10d977333d59..6f84ef68780d 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -30,17 +30,17 @@ # Note: google-api-core and google-auth are also included via transitive # dependency on google-cloud-bigquery, but this library also uses them # directly. - "google-api-core >= 2.10.2, <3.0.0dev", + "google-api-core >= 2.10.2, <3.0.0", "google-auth >=2.13.0", "google-auth-oauthlib >=0.7.0", # Please also update the minimum version in pandas_gbq/features.py to # allow pandas-gbq to detect invalid package versions at runtime. - "google-cloud-bigquery >=3.4.2,<4.0.0dev", + "google-cloud-bigquery >=3.4.2,<4.0.0", "packaging >=22.0.0", ] extras = { "bqstorage": [ - "google-cloud-bigquery-storage >=2.16.2, <3.0.0dev", + "google-cloud-bigquery-storage >=2.16.2, <3.0.0", ], "tqdm": ["tqdm>=4.23.0"], "geopandas": ["geopandas>=0.9.0", "Shapely>=1.8.4"], From fb322813e4ff1fc371b8cb9fbac9b7bd62242e57 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 10 Mar 2025 17:03:01 -0400 Subject: [PATCH 478/519] build: use python 3.10 for testing (#891) * build: use python 3.10 for testing * set default_python_version to 3.10 --- packages/pandas-gbq/.github/workflows/lint.yml | 2 +- packages/pandas-gbq/.github/workflows/unittest.yml | 2 +- packages/pandas-gbq/noxfile.py | 2 +- packages/pandas-gbq/owlbot.py | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/pandas-gbq/.github/workflows/lint.yml b/packages/pandas-gbq/.github/workflows/lint.yml index 4866193af2a9..1051da0bdda4 100644 --- a/packages/pandas-gbq/.github/workflows/lint.yml +++ b/packages/pandas-gbq/.github/workflows/lint.yml @@ -12,7 +12,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: "3.8" + python-version: "3.10" - name: Install nox run: | python -m pip install --upgrade setuptools pip wheel diff --git a/packages/pandas-gbq/.github/workflows/unittest.yml b/packages/pandas-gbq/.github/workflows/unittest.yml index ece9c4c6c0cf..107eac6b4b61 100644 --- a/packages/pandas-gbq/.github/workflows/unittest.yml +++ b/packages/pandas-gbq/.github/workflows/unittest.yml @@ -45,7 +45,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: "3.8" + python-version: "3.10" - name: Install coverage run: | python -m pip install --upgrade setuptools pip wheel diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index cf3405af6210..33923771e630 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -32,7 +32,7 @@ ISORT_VERSION = "isort==5.10.1" LINT_PATHS = ["docs", "pandas_gbq", "tests", "noxfile.py", "setup.py"] -DEFAULT_PYTHON_VERSION = "3.8" +DEFAULT_PYTHON_VERSION = "3.10" UNIT_TEST_PYTHON_VERSIONS = ["3.8", "3.9", "3.10", "3.11", "3.12"] diff --git a/packages/pandas-gbq/owlbot.py b/packages/pandas-gbq/owlbot.py index e50b9e9e6586..96a795c3f319 100644 --- a/packages/pandas-gbq/owlbot.py +++ b/packages/pandas-gbq/owlbot.py @@ -34,6 +34,7 @@ } extras = ["tqdm", "geopandas"] templated_files = common.py_library( + default_python_version="3.10", unit_test_python_versions=["3.8", "3.9", "3.10", "3.11", "3.12"], system_test_python_versions=["3.8", "3.9", "3.10", "3.11", "3.12"], cov_level=96, From c8c230bd80a431b8c35f852b838a7ae897bbdc62 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Wed, 12 Mar 2025 23:54:59 +0100 Subject: [PATCH 479/519] chore(deps): update all dependencies (#895) * chore(deps): update all dependencies * Update docs.yml * Update lint.yml --------- Co-authored-by: Lingqing Gan --- packages/pandas-gbq/samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 8e41d443d2b1..eba9e7659b3b 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,4 +1,4 @@ -google-cloud-bigquery-storage==2.28.0 +google-cloud-bigquery-storage==2.29.0 google-cloud-bigquery==3.30.0 pandas-gbq==0.28.0 pandas===2.0.3; python_version == '3.8' From b4f032e72d93561ea3aa684330b052bffe65eae5 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 14 Mar 2025 09:44:51 -0400 Subject: [PATCH 480/519] fix: remove setup.cfg configuration for creating universal wheels (#898) `setup.cfg` contains a setting to create a `Universal Wheel` which is only needed if libraries support both Python 2 and Python 3. This library only supports Python 3 so this setting is no longer needed. See https://packaging.python.org/en/latest/guides/distributing-packages-using-setuptools/#wheels. See similar PR https://togithub.com/googleapis/google-cloud-python/pull/13659 which includes this stack trace ``` running bdist_wheel /tmp/pip-build-env-9o_3w17v/overlay/lib/python3.13/site-packages/setuptools/_distutils/cmd.py:135: SetuptoolsDeprecationWarning: bdist_wheel.universal is deprecated !! ******************************************************************************** With Python 2.7 end-of-life, support for building universal wheels (i.e., wheels that support both Python 2 and Python 3) is being obviated. Please discontinue using this option, or if you still need it, file an issue with pypa/setuptools describing your use case. By 2025-Aug-30, you need to update your project and remove deprecated calls or your builds will no longer be supported. ******************************************************************************** !! ``` --- packages/pandas-gbq/setup.cfg | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 packages/pandas-gbq/setup.cfg diff --git a/packages/pandas-gbq/setup.cfg b/packages/pandas-gbq/setup.cfg deleted file mode 100644 index 052350089505..000000000000 --- a/packages/pandas-gbq/setup.cfg +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Generated by synthtool. DO NOT EDIT! -[bdist_wheel] -universal = 1 From e394247a3dcbac775ffdd527ce8a2a9640f5a1f4 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Thu, 27 Mar 2025 19:00:57 +0100 Subject: [PATCH 481/519] chore(deps): update all dependencies (#897) * chore(deps): update all dependencies * Update docs.yml * Update lint.yml --------- Co-authored-by: Lingqing Gan --- packages/pandas-gbq/samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index eba9e7659b3b..a451f10ac5b3 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,4 +1,4 @@ -google-cloud-bigquery-storage==2.29.0 +google-cloud-bigquery-storage==2.29.1 google-cloud-bigquery==3.30.0 pandas-gbq==0.28.0 pandas===2.0.3; python_version == '3.8' From 9ff623408b5807dfbfcd0128e9e675e12faad9d4 Mon Sep 17 00:00:00 2001 From: Lingqing Gan Date: Thu, 3 Apr 2025 11:40:53 -0700 Subject: [PATCH 482/519] fix: remove unnecessary global variable definition (#907) --- packages/pandas-gbq/pandas_gbq/gbq.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index bd3afb973be8..932c98feeec5 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -271,7 +271,6 @@ def __init__( rfc9110_delimiter=False, bigquery_client=None, ): - global context from google.api_core.exceptions import ClientError, GoogleAPIError from pandas_gbq import auth @@ -869,8 +868,6 @@ def read_gbq( df: DataFrame DataFrame representing results of query. """ - global context - if dialect is None: dialect = context.dialect From 6a825ed4dff9972498028307c39b1d7442cc2c5b Mon Sep 17 00:00:00 2001 From: Shenyang Cai Date: Mon, 28 Apr 2025 11:09:24 -0700 Subject: [PATCH 483/519] refactor: break down gbq.py file to several smaller ones (#909) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor: break down gbq.py file to several smaller ones * fix test failures * fix more test import failures * fix prerelease tests * fix system tests * restore imports * restore imports * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * add noqa for backward compatibility * fix lint * fix test failures by ignoring timestamp granularity: --------- Co-authored-by: Owl Bot --- packages/pandas-gbq/pandas_gbq/__init__.py | 3 +- packages/pandas-gbq/pandas_gbq/contexts.py | 120 +++ packages/pandas-gbq/pandas_gbq/exceptions.py | 64 ++ packages/pandas-gbq/pandas_gbq/gbq.py | 711 +----------------- .../pandas-gbq/pandas_gbq/gbq_connector.py | 527 +++++++++++++ .../pandas-gbq/tests/system/test_read_gbq.py | 4 +- 6 files changed, 736 insertions(+), 693 deletions(-) create mode 100644 packages/pandas-gbq/pandas_gbq/contexts.py create mode 100644 packages/pandas-gbq/pandas_gbq/gbq_connector.py diff --git a/packages/pandas-gbq/pandas_gbq/__init__.py b/packages/pandas-gbq/pandas_gbq/__init__.py index 76c33d60adde..184f8c4416f7 100644 --- a/packages/pandas-gbq/pandas_gbq/__init__.py +++ b/packages/pandas-gbq/pandas_gbq/__init__.py @@ -5,9 +5,10 @@ import warnings from pandas_gbq import version as pandas_gbq_version +from pandas_gbq.contexts import Context, context from . import _versions_helpers -from .gbq import Context, context, read_gbq, to_gbq # noqa +from .gbq import read_gbq, to_gbq # noqa sys_major, sys_minor, sys_micro = _versions_helpers.extract_runtime_version() if sys_major == 3 and sys_minor in (7, 8): diff --git a/packages/pandas-gbq/pandas_gbq/contexts.py b/packages/pandas-gbq/pandas_gbq/contexts.py new file mode 100644 index 000000000000..76a5a1e2efda --- /dev/null +++ b/packages/pandas-gbq/pandas_gbq/contexts.py @@ -0,0 +1,120 @@ +# Copyright (c) 2025 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + + +class Context(object): + """Storage for objects to be used throughout a session. + + A Context object is initialized when the ``pandas_gbq`` module is + imported, and can be found at :attr:`pandas_gbq.context`. + """ + + def __init__(self): + self._credentials = None + self._project = None + # dialect defaults to None so that read_gbq can stop warning if set. + self._dialect = None + + @property + def credentials(self): + """ + Credentials to use for Google APIs. + + These credentials are automatically cached in memory by calls to + :func:`pandas_gbq.read_gbq` and :func:`pandas_gbq.to_gbq`. To + manually set the credentials, construct an + :class:`google.auth.credentials.Credentials` object and set it as + the context credentials as demonstrated in the example below. See + `auth docs`_ for more information on obtaining credentials. + + .. _auth docs: http://google-auth.readthedocs.io + /en/latest/user-guide.html#obtaining-credentials + + Returns + ------- + google.auth.credentials.Credentials + + Examples + -------- + + Manually setting the context credentials: + + >>> import pandas_gbq + >>> from google.oauth2 import service_account + >>> credentials = service_account.Credentials.from_service_account_file( + ... '/path/to/key.json', + ... ) + >>> pandas_gbq.context.credentials = credentials + """ + return self._credentials + + @credentials.setter + def credentials(self, value): + self._credentials = value + + @property + def project(self): + """Default project to use for calls to Google APIs. + + Returns + ------- + str + + Examples + -------- + + Manually setting the context project: + + >>> import pandas_gbq + >>> pandas_gbq.context.project = 'my-project' + """ + return self._project + + @project.setter + def project(self, value): + self._project = value + + @property + def dialect(self): + """ + Default dialect to use in :func:`pandas_gbq.read_gbq`. + + Allowed values for the BigQuery SQL syntax dialect: + + ``'legacy'`` + Use BigQuery's legacy SQL dialect. For more information see + `BigQuery Legacy SQL Reference + `__. + ``'standard'`` + Use BigQuery's standard SQL, which is + compliant with the SQL 2011 standard. For more information + see `BigQuery Standard SQL Reference + `__. + + Returns + ------- + str + + Examples + -------- + + Setting the default syntax to standard: + + >>> import pandas_gbq + >>> pandas_gbq.context.dialect = 'standard' + """ + return self._dialect + + @dialect.setter + def dialect(self, value): + self._dialect = value + + +# Create an empty context, used to cache credentials. +context = Context() +"""A :class:`pandas_gbq.Context` object used to cache credentials. + +Credentials automatically are cached in-memory by :func:`pandas_gbq.read_gbq` +and :func:`pandas_gbq.to_gbq`. +""" diff --git a/packages/pandas-gbq/pandas_gbq/exceptions.py b/packages/pandas-gbq/pandas_gbq/exceptions.py index af58212ee91f..1acec712ebc6 100644 --- a/packages/pandas-gbq/pandas_gbq/exceptions.py +++ b/packages/pandas-gbq/pandas_gbq/exceptions.py @@ -3,6 +3,70 @@ # license that can be found in the LICENSE file. +class DatasetCreationError(ValueError): + """ + Raised when the create dataset method fails + """ + + +class InvalidColumnOrder(ValueError): + """ + Raised when the provided column order for output + results DataFrame does not match the schema + returned by BigQuery. + """ + + +class InvalidIndexColumn(ValueError): + """ + Raised when the provided index column for output + results DataFrame does not match the schema + returned by BigQuery. + """ + + +class InvalidPageToken(ValueError): + """ + Raised when Google BigQuery fails to return, + or returns a duplicate page token. + """ + + +class InvalidSchema(ValueError): + """ + Raised when the provided DataFrame does + not match the schema of the destination + table in BigQuery. + """ + + def __init__(self, message: str): + self._message = message + + @property + def message(self) -> str: + return self._message + + +class NotFoundException(ValueError): + """ + Raised when the project_id, table or dataset provided in the query could + not be found. + """ + + +class TableCreationError(ValueError): + """ + Raised when the create table method fails + """ + + def __init__(self, message: str): + self._message = message + + @property + def message(self) -> str: + return self._message + + class GenericGBQException(ValueError): """ Raised when an unrecognized Google API Error occurs. diff --git a/packages/pandas-gbq/pandas_gbq/gbq.py b/packages/pandas-gbq/pandas_gbq/gbq.py index 932c98feeec5..8db1d4ea8984 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq.py +++ b/packages/pandas-gbq/pandas_gbq/gbq.py @@ -6,32 +6,31 @@ from datetime import datetime import logging import re -import time -import typing -from typing import Any, Dict, Optional, Sequence, Union import warnings -import numpy as np - -# Only import at module-level at type checking time to avoid circular -# dependencies in the pandas package, which has an optional dependency on -# pandas-gbq. -if typing.TYPE_CHECKING: # pragma: NO COVER - import pandas - -import pandas_gbq.constants -import pandas_gbq.exceptions -from pandas_gbq.exceptions import GenericGBQException, QueryTimeout +from pandas_gbq.contexts import Context # noqa - backward compatible export +from pandas_gbq.contexts import context +from pandas_gbq.exceptions import ( # noqa - backward compatible export + DatasetCreationError, + GenericGBQException, + InvalidColumnOrder, + InvalidIndexColumn, + NotFoundException, + TableCreationError, +) +from pandas_gbq.exceptions import InvalidPageToken # noqa - backward compatible export +from pandas_gbq.exceptions import InvalidSchema # noqa - backward compatible export +from pandas_gbq.exceptions import QueryTimeout # noqa - backward compatible export from pandas_gbq.features import FEATURES -import pandas_gbq.query +from pandas_gbq.gbq_connector import ( # noqa - backward compatible export + GbqConnector, + _bqschema_to_nullsafe_dtypes, + _finalize_dtypes, + create_user_agent, +) +from pandas_gbq.gbq_connector import _get_client # noqa - backward compatible export import pandas_gbq.schema import pandas_gbq.schema.pandas_to_bigquery -import pandas_gbq.timestamp - -try: - import tqdm # noqa -except ImportError: - tqdm = None logger = logging.getLogger(__name__) @@ -72,601 +71,6 @@ def _is_query(query_or_table: str) -> bool: return re.search(r"\s", query_or_table.strip(), re.MULTILINE) is not None -class DatasetCreationError(ValueError): - """ - Raised when the create dataset method fails - """ - - -class InvalidColumnOrder(ValueError): - """ - Raised when the provided column order for output - results DataFrame does not match the schema - returned by BigQuery. - """ - - -class InvalidIndexColumn(ValueError): - """ - Raised when the provided index column for output - results DataFrame does not match the schema - returned by BigQuery. - """ - - -class InvalidPageToken(ValueError): - """ - Raised when Google BigQuery fails to return, - or returns a duplicate page token. - """ - - -class InvalidSchema(ValueError): - """ - Raised when the provided DataFrame does - not match the schema of the destination - table in BigQuery. - """ - - def __init__(self, message: str): - self._message = message - - @property - def message(self) -> str: - return self._message - - -class NotFoundException(ValueError): - """ - Raised when the project_id, table or dataset provided in the query could - not be found. - """ - - -class TableCreationError(ValueError): - """ - Raised when the create table method fails - """ - - def __init__(self, message: str): - self._message = message - - @property - def message(self) -> str: - return self._message - - -class Context(object): - """Storage for objects to be used throughout a session. - - A Context object is initialized when the ``pandas_gbq`` module is - imported, and can be found at :attr:`pandas_gbq.context`. - """ - - def __init__(self): - self._credentials = None - self._project = None - # dialect defaults to None so that read_gbq can stop warning if set. - self._dialect = None - - @property - def credentials(self): - """ - Credentials to use for Google APIs. - - These credentials are automatically cached in memory by calls to - :func:`pandas_gbq.read_gbq` and :func:`pandas_gbq.to_gbq`. To - manually set the credentials, construct an - :class:`google.auth.credentials.Credentials` object and set it as - the context credentials as demonstrated in the example below. See - `auth docs`_ for more information on obtaining credentials. - - .. _auth docs: http://google-auth.readthedocs.io - /en/latest/user-guide.html#obtaining-credentials - - Returns - ------- - google.auth.credentials.Credentials - - Examples - -------- - - Manually setting the context credentials: - - >>> import pandas_gbq - >>> from google.oauth2 import service_account - >>> credentials = service_account.Credentials.from_service_account_file( - ... '/path/to/key.json', - ... ) - >>> pandas_gbq.context.credentials = credentials - """ - return self._credentials - - @credentials.setter - def credentials(self, value): - self._credentials = value - - @property - def project(self): - """Default project to use for calls to Google APIs. - - Returns - ------- - str - - Examples - -------- - - Manually setting the context project: - - >>> import pandas_gbq - >>> pandas_gbq.context.project = 'my-project' - """ - return self._project - - @project.setter - def project(self, value): - self._project = value - - @property - def dialect(self): - """ - Default dialect to use in :func:`pandas_gbq.read_gbq`. - - Allowed values for the BigQuery SQL syntax dialect: - - ``'legacy'`` - Use BigQuery's legacy SQL dialect. For more information see - `BigQuery Legacy SQL Reference - `__. - ``'standard'`` - Use BigQuery's standard SQL, which is - compliant with the SQL 2011 standard. For more information - see `BigQuery Standard SQL Reference - `__. - - Returns - ------- - str - - Examples - -------- - - Setting the default syntax to standard: - - >>> import pandas_gbq - >>> pandas_gbq.context.dialect = 'standard' - """ - return self._dialect - - @dialect.setter - def dialect(self, value): - self._dialect = value - - -# Create an empty context, used to cache credentials. -context = Context() -"""A :class:`pandas_gbq.Context` object used to cache credentials. - -Credentials automatically are cached in-memory by :func:`pandas_gbq.read_gbq` -and :func:`pandas_gbq.to_gbq`. -""" - - -class GbqConnector(object): - def __init__( - self, - project_id, - reauth=False, - private_key=None, - auth_local_webserver=True, - dialect="standard", - location=None, - credentials=None, - use_bqstorage_api=False, - auth_redirect_uri=None, - client_id=None, - client_secret=None, - user_agent=None, - rfc9110_delimiter=False, - bigquery_client=None, - ): - from google.api_core.exceptions import ClientError, GoogleAPIError - - from pandas_gbq import auth - - self.http_error = (ClientError, GoogleAPIError) - self.project_id = project_id - self.location = location - self.reauth = reauth - self.private_key = private_key - self.auth_local_webserver = auth_local_webserver - self.dialect = dialect - self.credentials = credentials - self.auth_redirect_uri = auth_redirect_uri - self.client_id = client_id - self.client_secret = client_secret - self.user_agent = user_agent - self.rfc9110_delimiter = rfc9110_delimiter - self.use_bqstorage_api = use_bqstorage_api - - if bigquery_client is not None: - # If a bq client is already provided, use it to populate auth fields. - self.project_id = bigquery_client.project - self.credentials = bigquery_client._credentials - self.client = bigquery_client - return - - default_project = None - - # Service account credentials have a project associated with them. - # Prefer that project if none was supplied. - if self.project_id is None and hasattr(self.credentials, "project_id"): - self.project_id = credentials.project_id - - # Load credentials from cache. - if not self.credentials: - self.credentials = context.credentials - default_project = context.project - - # Credentials were explicitly asked for, so don't use the cache. - if private_key or reauth or not self.credentials: - self.credentials, default_project = auth.get_credentials( - private_key=private_key, - project_id=project_id, - reauth=reauth, - auth_local_webserver=auth_local_webserver, - auth_redirect_uri=auth_redirect_uri, - client_id=client_id, - client_secret=client_secret, - ) - - if self.project_id is None: - self.project_id = default_project - - if self.project_id is None: - raise ValueError("Could not determine project ID and one was not supplied.") - - # Cache the credentials if they haven't been set yet. - if context.credentials is None: - context.credentials = self.credentials - if context.project is None: - context.project = self.project_id - - self.client = _get_client( - self.user_agent, self.rfc9110_delimiter, self.project_id, self.credentials - ) - - def _start_timer(self): - self.start = time.time() - - def get_elapsed_seconds(self): - return round(time.time() - self.start, 2) - - def log_elapsed_seconds(self, prefix="Elapsed", postfix="s.", overlong=6): - sec = self.get_elapsed_seconds() - if sec > overlong: - logger.info("{} {} {}".format(prefix, sec, postfix)) - - def get_client(self): - import google.api_core.client_info - - bigquery = FEATURES.bigquery_try_import() - - user_agent = create_user_agent( - user_agent=self.user_agent, rfc9110_delimiter=self.rfc9110_delimiter - ) - - client_info = google.api_core.client_info.ClientInfo( - user_agent=user_agent, - ) - return bigquery.Client( - project=self.project_id, - credentials=self.credentials, - client_info=client_info, - ) - - @staticmethod - def process_http_error(ex): - # See `BigQuery Troubleshooting Errors - # `__ - - message = ( - ex.message.casefold() - if hasattr(ex, "message") and ex.message is not None - else "" - ) - if "cancelled" in message: - raise QueryTimeout("Reason: {0}".format(ex)) - elif "schema does not match" in message: - error_message = ex.errors[0]["message"] - raise InvalidSchema(f"Reason: {error_message}") - elif "already exists: table" in message: - error_message = ex.errors[0]["message"] - raise TableCreationError(f"Reason: {error_message}") - else: - raise GenericGBQException("Reason: {0}".format(ex)) from ex - - def download_table( - self, - table_id: str, - max_results: Optional[int] = None, - progress_bar_type: Optional[str] = None, - dtypes: Optional[Dict[str, Union[str, Any]]] = None, - ) -> "pandas.DataFrame": - from google.cloud import bigquery - - self._start_timer() - - try: - table_ref = bigquery.TableReference.from_string( - table_id, default_project=self.project_id - ) - rows_iter = self.client.list_rows(table_ref, max_results=max_results) - except self.http_error as ex: - self.process_http_error(ex) - - return self._download_results( - rows_iter, - max_results=max_results, - progress_bar_type=progress_bar_type, - user_dtypes=dtypes, - ) - - def run_query(self, query, max_results=None, progress_bar_type=None, **kwargs): - from google.cloud import bigquery - - job_config_dict = { - "query": { - "useLegacySql": self.dialect - == "legacy" - # 'allowLargeResults', 'createDisposition', - # 'preserveNulls', destinationTable, useQueryCache - } - } - config = kwargs.get("configuration") - if config is not None: - job_config_dict.update(config) - - timeout_ms = job_config_dict.get("jobTimeoutMs") or job_config_dict[ - "query" - ].get("timeoutMs") - - if timeout_ms: - timeout_ms = int(timeout_ms) - # Having too small a timeout_ms results in individual - # API calls timing out before they can finish. - # ~300 milliseconds is rule of thumb for bare minimum - # latency from the BigQuery API, however, 400 milliseconds - # produced too many issues with flakybot failures. - minimum_latency = 500 - if timeout_ms < minimum_latency: - raise QueryTimeout( - f"Query timeout must be at least 500 milliseconds: timeout_ms equals {timeout_ms}." - ) - else: - timeout_ms = None - - self._start_timer() - job_config = bigquery.QueryJobConfig.from_api_repr(job_config_dict) - - if FEATURES.bigquery_has_query_and_wait: - rows_iter = pandas_gbq.query.query_and_wait_via_client_library( - self, - self.client, - query, - location=self.location, - project_id=self.project_id, - job_config=job_config, - max_results=max_results, - timeout_ms=timeout_ms, - ) - else: - rows_iter = pandas_gbq.query.query_and_wait( - self, - self.client, - query, - location=self.location, - project_id=self.project_id, - job_config=job_config, - max_results=max_results, - timeout_ms=timeout_ms, - ) - - dtypes = kwargs.get("dtypes") - return self._download_results( - rows_iter, - max_results=max_results, - progress_bar_type=progress_bar_type, - user_dtypes=dtypes, - ) - - def _download_results( - self, - rows_iter, - max_results=None, - progress_bar_type=None, - user_dtypes=None, - ): - # No results are desired, so don't bother downloading anything. - if max_results == 0: - return None - - if user_dtypes is None: - user_dtypes = {} - - create_bqstorage_client = self.use_bqstorage_api - if max_results is not None: - create_bqstorage_client = False - - # If we're downloading a large table, BigQuery DataFrames might be a - # better fit. Not all code paths will populate rows_iter._table, but - # if it's not populated that means we are working with a small result - # set. - if (table_ref := getattr(rows_iter, "_table", None)) is not None: - table = self.client.get_table(table_ref) - if ( - isinstance((num_bytes := table.num_bytes), int) - and num_bytes > pandas_gbq.constants.BYTES_TO_RECOMMEND_BIGFRAMES - ): - num_gib = num_bytes / pandas_gbq.constants.BYTES_IN_GIB - warnings.warn( - f"Recommendation: Your results are {num_gib:.1f} GiB. " - "Consider using BigQuery DataFrames (https://bit.ly/bigframes-intro)" - "to process large results with pandas compatible APIs with transparent SQL " - "pushdown to BigQuery engine. This provides an opportunity to save on costs " - "and improve performance. " - "Please reach out to bigframes-feedback@google.com with any " - "questions or concerns. To disable this message, run " - "warnings.simplefilter('ignore', category=pandas_gbq.exceptions.LargeResultsWarning)", - category=pandas_gbq.exceptions.LargeResultsWarning, - # user's code - # -> read_gbq - # -> run_query - # -> download_results - stacklevel=4, - ) - - try: - schema_fields = [field.to_api_repr() for field in rows_iter.schema] - conversion_dtypes = _bqschema_to_nullsafe_dtypes(schema_fields) - conversion_dtypes.update(user_dtypes) - df = rows_iter.to_dataframe( - dtypes=conversion_dtypes, - progress_bar_type=progress_bar_type, - create_bqstorage_client=create_bqstorage_client, - ) - except self.http_error as ex: - self.process_http_error(ex) - - df = _finalize_dtypes(df, schema_fields) - - logger.debug("Got {} rows.\n".format(rows_iter.total_rows)) - return df - - def load_data( - self, - dataframe, - destination_table_ref, - write_disposition, - chunksize=None, - schema=None, - progress_bar=True, - api_method: str = "load_parquet", - billing_project: Optional[str] = None, - ): - from pandas_gbq import load - - total_rows = len(dataframe) - - try: - chunks = load.load_chunks( - self.client, - dataframe, - destination_table_ref, - chunksize=chunksize, - schema=schema, - location=self.location, - api_method=api_method, - write_disposition=write_disposition, - billing_project=billing_project, - ) - if progress_bar and tqdm: - chunks = tqdm.tqdm(chunks) - for remaining_rows in chunks: - logger.info( - "\r{} out of {} rows loaded.".format( - total_rows - remaining_rows, total_rows - ) - ) - except self.http_error as ex: - self.process_http_error(ex) - - -def _bqschema_to_nullsafe_dtypes(schema_fields): - """Specify explicit dtypes based on BigQuery schema. - - This function only specifies a dtype when the dtype allows nulls. - Otherwise, use pandas's default dtype choice. - - See: http://pandas.pydata.org/pandas-docs/dev/missing_data.html - #missing-data-casting-rules-and-indexing - """ - import db_dtypes - - # If you update this mapping, also update the table at - # `docs/reading.rst`. - dtype_map = { - "FLOAT": np.dtype(float), - "INTEGER": "Int64", - "TIME": db_dtypes.TimeDtype(), - # Note: Other types such as 'datetime64[ns]' and db_types.DateDtype() - # are not included because the pandas range does not align with the - # BigQuery range. We need to attempt a conversion to those types and - # fall back to 'object' when there are out-of-range values. - } - - # Amend dtype_map with newer extension types if pandas version allows. - if FEATURES.pandas_has_boolean_dtype: - dtype_map["BOOLEAN"] = "boolean" - - dtypes = {} - for field in schema_fields: - name = str(field["name"]) - # Array BigQuery type is represented as an object column containing - # list objects. - if field["mode"].upper() == "REPEATED": - dtypes[name] = "object" - continue - - dtype = dtype_map.get(field["type"].upper()) - if dtype: - dtypes[name] = dtype - - return dtypes - - -def _finalize_dtypes( - df: "pandas.DataFrame", schema_fields: Sequence[Dict[str, Any]] -) -> "pandas.DataFrame": - """ - Attempt to change the dtypes of those columns that don't map exactly. - - For example db_dtypes.DateDtype() and datetime64[ns] cannot represent - 0001-01-01, but they can represent dates within a couple hundred years of - 1970. See: - https://github.com/googleapis/python-bigquery-pandas/issues/365 - """ - import db_dtypes - import pandas.api.types - - # If you update this mapping, also update the table at - # `docs/reading.rst`. - dtype_map = { - "DATE": db_dtypes.DateDtype(), - "DATETIME": "datetime64[ns]", - "TIMESTAMP": "datetime64[ns]", - } - - for field in schema_fields: - # This method doesn't modify ARRAY/REPEATED columns. - if field["mode"].upper() == "REPEATED": - continue - - name = str(field["name"]) - dtype = dtype_map.get(field["type"].upper()) - - # Avoid deprecated conversion to timezone-naive dtype by only casting - # object dtypes. - if dtype and pandas.api.types.is_object_dtype(df[name]): - df[name] = df[name].astype(dtype, errors="ignore") - - # Ensure any TIMESTAMP columns are tz-aware. - df = pandas_gbq.timestamp.localize_df(df, schema_fields) - - return df - - def _transform_read_gbq_configuration(configuration): """ For backwards-compatibility, convert any previously client-side only @@ -1453,78 +857,3 @@ def create(self, dataset_id): self.client.create_dataset(dataset) except self.http_error as ex: self.process_http_error(ex) - - -def create_user_agent( - user_agent: Optional[str] = None, rfc9110_delimiter: bool = False -) -> str: - """Creates a user agent string. - - The legacy format of our the user agent string was: `product-x.y.z` (where x, - y, and z are the major, minor, and micro version numbers). - - Users are able to prepend this string with their own user agent identifier - to render something similar to ` pandas-x.y.z`. - - The legacy format used a hyphen to separate the product from the product - version which differs slightly from the format recommended by RFC9110, which is: - `product/x.y.z`. To produce a user agent more in line with the RFC, set - rfc9110_delimiter to True. This setting does not depend on whether a - user_agent is also supplied. - - Reference: - https://www.rfc-editor.org/info/rfc9110 - - Args: - user_agent (Optional[str]): User agent string. - - rfc9110_delimiter (Optional[bool]): Sets delimiter to a hyphen or a slash. - Default is False, meaning a hyphen will be used. - - Returns (str): - Customized user agent string. - - Deprecation Warning: - In a future major release, the default delimiter will be changed to - a `/` in accordance with RFC9110. - """ - import pandas as pd - - if rfc9110_delimiter: - delimiter = "/" - else: - warnings.warn( - "In a future major release, the default delimiter will be " - "changed to a `/` in accordance with RFC9110.", - PendingDeprecationWarning, - stacklevel=2, - ) - delimiter = "-" - - identity = f"pandas{delimiter}{pd.__version__}" - - if user_agent is None: - user_agent = identity - else: - user_agent = f"{user_agent} {identity}" - - return user_agent - - -def _get_client(user_agent, rfc9110_delimiter, project_id, credentials): - import google.api_core.client_info - - bigquery = FEATURES.bigquery_try_import() - - user_agent = create_user_agent( - user_agent=user_agent, rfc9110_delimiter=rfc9110_delimiter - ) - - client_info = google.api_core.client_info.ClientInfo( - user_agent=user_agent, - ) - return bigquery.Client( - project=project_id, - credentials=credentials, - client_info=client_info, - ) diff --git a/packages/pandas-gbq/pandas_gbq/gbq_connector.py b/packages/pandas-gbq/pandas_gbq/gbq_connector.py new file mode 100644 index 000000000000..97a22db4a8d7 --- /dev/null +++ b/packages/pandas-gbq/pandas_gbq/gbq_connector.py @@ -0,0 +1,527 @@ +# Copyright (c) 2025 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + + +import logging +import time +import typing +from typing import Any, Dict, Optional, Sequence, Union +import warnings + +import numpy as np + +# Only import at module-level at type checking time to avoid circular +# dependencies in the pandas package, which has an optional dependency on +# pandas-gbq. +if typing.TYPE_CHECKING: # pragma: NO COVER + import pandas + +import pandas_gbq.constants +from pandas_gbq.contexts import context +import pandas_gbq.exceptions +from pandas_gbq.exceptions import ( + GenericGBQException, + InvalidSchema, + QueryTimeout, + TableCreationError, +) +from pandas_gbq.features import FEATURES +import pandas_gbq.query +import pandas_gbq.timestamp + +try: + import tqdm # noqa +except ImportError: + tqdm = None + +logger = logging.getLogger(__name__) + + +class GbqConnector: + def __init__( + self, + project_id, + reauth=False, + private_key=None, + auth_local_webserver=True, + dialect="standard", + location=None, + credentials=None, + use_bqstorage_api=False, + auth_redirect_uri=None, + client_id=None, + client_secret=None, + user_agent=None, + rfc9110_delimiter=False, + bigquery_client=None, + ): + from google.api_core.exceptions import ClientError, GoogleAPIError + + from pandas_gbq import auth + + self.http_error = (ClientError, GoogleAPIError) + self.project_id = project_id + self.location = location + self.reauth = reauth + self.private_key = private_key + self.auth_local_webserver = auth_local_webserver + self.dialect = dialect + self.credentials = credentials + self.auth_redirect_uri = auth_redirect_uri + self.client_id = client_id + self.client_secret = client_secret + self.user_agent = user_agent + self.rfc9110_delimiter = rfc9110_delimiter + self.use_bqstorage_api = use_bqstorage_api + + if bigquery_client is not None: + # If a bq client is already provided, use it to populate auth fields. + self.project_id = bigquery_client.project + self.credentials = bigquery_client._credentials + self.client = bigquery_client + return + + default_project = None + + # Service account credentials have a project associated with them. + # Prefer that project if none was supplied. + if self.project_id is None and hasattr(self.credentials, "project_id"): + self.project_id = credentials.project_id + + # Load credentials from cache. + if not self.credentials: + self.credentials = context.credentials + default_project = context.project + + # Credentials were explicitly asked for, so don't use the cache. + if private_key or reauth or not self.credentials: + self.credentials, default_project = auth.get_credentials( + private_key=private_key, + project_id=project_id, + reauth=reauth, + auth_local_webserver=auth_local_webserver, + auth_redirect_uri=auth_redirect_uri, + client_id=client_id, + client_secret=client_secret, + ) + + if self.project_id is None: + self.project_id = default_project + + if self.project_id is None: + raise ValueError("Could not determine project ID and one was not supplied.") + + # Cache the credentials if they haven't been set yet. + if context.credentials is None: + context.credentials = self.credentials + if context.project is None: + context.project = self.project_id + + self.client = _get_client( + self.user_agent, self.rfc9110_delimiter, self.project_id, self.credentials + ) + + def _start_timer(self): + self.start = time.time() + + def get_elapsed_seconds(self): + return round(time.time() - self.start, 2) + + def log_elapsed_seconds(self, prefix="Elapsed", postfix="s.", overlong=6): + sec = self.get_elapsed_seconds() + if sec > overlong: + logger.info("{} {} {}".format(prefix, sec, postfix)) + + def get_client(self): + import google.api_core.client_info + + bigquery = FEATURES.bigquery_try_import() + + user_agent = create_user_agent( + user_agent=self.user_agent, rfc9110_delimiter=self.rfc9110_delimiter + ) + + client_info = google.api_core.client_info.ClientInfo( + user_agent=user_agent, + ) + return bigquery.Client( + project=self.project_id, + credentials=self.credentials, + client_info=client_info, + ) + + @staticmethod + def process_http_error(ex): + # See `BigQuery Troubleshooting Errors + # `__ + + message = ( + ex.message.casefold() + if hasattr(ex, "message") and ex.message is not None + else "" + ) + if "cancelled" in message: + raise QueryTimeout("Reason: {0}".format(ex)) + elif "schema does not match" in message: + error_message = ex.errors[0]["message"] + raise InvalidSchema(f"Reason: {error_message}") + elif "already exists: table" in message: + error_message = ex.errors[0]["message"] + raise TableCreationError(f"Reason: {error_message}") + else: + raise GenericGBQException("Reason: {0}".format(ex)) from ex + + def download_table( + self, + table_id: str, + max_results: Optional[int] = None, + progress_bar_type: Optional[str] = None, + dtypes: Optional[Dict[str, Union[str, Any]]] = None, + ) -> "pandas.DataFrame": + from google.cloud import bigquery + + self._start_timer() + + try: + table_ref = bigquery.TableReference.from_string( + table_id, default_project=self.project_id + ) + rows_iter = self.client.list_rows(table_ref, max_results=max_results) + except self.http_error as ex: + self.process_http_error(ex) + + return self._download_results( + rows_iter, + max_results=max_results, + progress_bar_type=progress_bar_type, + user_dtypes=dtypes, + ) + + def run_query(self, query, max_results=None, progress_bar_type=None, **kwargs): + from google.cloud import bigquery + + job_config_dict = { + "query": { + "useLegacySql": self.dialect + == "legacy" + # 'allowLargeResults', 'createDisposition', + # 'preserveNulls', destinationTable, useQueryCache + } + } + config = kwargs.get("configuration") + if config is not None: + job_config_dict.update(config) + + timeout_ms = job_config_dict.get("jobTimeoutMs") or job_config_dict[ + "query" + ].get("timeoutMs") + + if timeout_ms: + timeout_ms = int(timeout_ms) + # Having too small a timeout_ms results in individual + # API calls timing out before they can finish. + # ~300 milliseconds is rule of thumb for bare minimum + # latency from the BigQuery API, however, 400 milliseconds + # produced too many issues with flakybot failures. + minimum_latency = 500 + if timeout_ms < minimum_latency: + raise QueryTimeout( + f"Query timeout must be at least 500 milliseconds: timeout_ms equals {timeout_ms}." + ) + else: + timeout_ms = None + + self._start_timer() + job_config = bigquery.QueryJobConfig.from_api_repr(job_config_dict) + + if FEATURES.bigquery_has_query_and_wait: + rows_iter = pandas_gbq.query.query_and_wait_via_client_library( + self, + self.client, + query, + location=self.location, + project_id=self.project_id, + job_config=job_config, + max_results=max_results, + timeout_ms=timeout_ms, + ) + else: + rows_iter = pandas_gbq.query.query_and_wait( + self, + self.client, + query, + location=self.location, + project_id=self.project_id, + job_config=job_config, + max_results=max_results, + timeout_ms=timeout_ms, + ) + + dtypes = kwargs.get("dtypes") + return self._download_results( + rows_iter, + max_results=max_results, + progress_bar_type=progress_bar_type, + user_dtypes=dtypes, + ) + + def _download_results( + self, + rows_iter, + max_results=None, + progress_bar_type=None, + user_dtypes=None, + ): + # No results are desired, so don't bother downloading anything. + if max_results == 0: + return None + + if user_dtypes is None: + user_dtypes = {} + + create_bqstorage_client = self.use_bqstorage_api + if max_results is not None: + create_bqstorage_client = False + + # If we're downloading a large table, BigQuery DataFrames might be a + # better fit. Not all code paths will populate rows_iter._table, but + # if it's not populated that means we are working with a small result + # set. + if (table_ref := getattr(rows_iter, "_table", None)) is not None: + table = self.client.get_table(table_ref) + if ( + isinstance((num_bytes := table.num_bytes), int) + and num_bytes > pandas_gbq.constants.BYTES_TO_RECOMMEND_BIGFRAMES + ): + num_gib = num_bytes / pandas_gbq.constants.BYTES_IN_GIB + warnings.warn( + f"Recommendation: Your results are {num_gib:.1f} GiB. " + "Consider using BigQuery DataFrames (https://bit.ly/bigframes-intro)" + "to process large results with pandas compatible APIs with transparent SQL " + "pushdown to BigQuery engine. This provides an opportunity to save on costs " + "and improve performance. " + "Please reach out to bigframes-feedback@google.com with any " + "questions or concerns. To disable this message, run " + "warnings.simplefilter('ignore', category=pandas_gbq.exceptions.LargeResultsWarning)", + category=pandas_gbq.exceptions.LargeResultsWarning, + # user's code + # -> read_gbq + # -> run_query + # -> download_results + stacklevel=4, + ) + + try: + schema_fields = [field.to_api_repr() for field in rows_iter.schema] + conversion_dtypes = _bqschema_to_nullsafe_dtypes(schema_fields) + conversion_dtypes.update(user_dtypes) + df = rows_iter.to_dataframe( + dtypes=conversion_dtypes, + progress_bar_type=progress_bar_type, + create_bqstorage_client=create_bqstorage_client, + ) + except self.http_error as ex: + self.process_http_error(ex) + + df = _finalize_dtypes(df, schema_fields) + + logger.debug("Got {} rows.\n".format(rows_iter.total_rows)) + return df + + def load_data( + self, + dataframe, + destination_table_ref, + write_disposition, + chunksize=None, + schema=None, + progress_bar=True, + api_method: str = "load_parquet", + billing_project: Optional[str] = None, + ): + from pandas_gbq import load + + total_rows = len(dataframe) + + try: + chunks = load.load_chunks( + self.client, + dataframe, + destination_table_ref, + chunksize=chunksize, + schema=schema, + location=self.location, + api_method=api_method, + write_disposition=write_disposition, + billing_project=billing_project, + ) + if progress_bar and tqdm: + chunks = tqdm.tqdm(chunks) + for remaining_rows in chunks: + logger.info( + "\r{} out of {} rows loaded.".format( + total_rows - remaining_rows, total_rows + ) + ) + except self.http_error as ex: + self.process_http_error(ex) + + +def _bqschema_to_nullsafe_dtypes(schema_fields): + """Specify explicit dtypes based on BigQuery schema. + + This function only specifies a dtype when the dtype allows nulls. + Otherwise, use pandas's default dtype choice. + + See: http://pandas.pydata.org/pandas-docs/dev/missing_data.html + #missing-data-casting-rules-and-indexing + """ + import db_dtypes + + # If you update this mapping, also update the table at + # `docs/reading.rst`. + dtype_map = { + "FLOAT": np.dtype(float), + "INTEGER": "Int64", + "TIME": db_dtypes.TimeDtype(), + # Note: Other types such as 'datetime64[ns]' and db_types.DateDtype() + # are not included because the pandas range does not align with the + # BigQuery range. We need to attempt a conversion to those types and + # fall back to 'object' when there are out-of-range values. + } + + # Amend dtype_map with newer extension types if pandas version allows. + if FEATURES.pandas_has_boolean_dtype: + dtype_map["BOOLEAN"] = "boolean" + + dtypes = {} + for field in schema_fields: + name = str(field["name"]) + # Array BigQuery type is represented as an object column containing + # list objects. + if field["mode"].upper() == "REPEATED": + dtypes[name] = "object" + continue + + dtype = dtype_map.get(field["type"].upper()) + if dtype: + dtypes[name] = dtype + + return dtypes + + +def _finalize_dtypes( + df: "pandas.DataFrame", schema_fields: Sequence[Dict[str, Any]] +) -> "pandas.DataFrame": + """ + Attempt to change the dtypes of those columns that don't map exactly. + + For example db_dtypes.DateDtype() and datetime64[ns] cannot represent + 0001-01-01, but they can represent dates within a couple hundred years of + 1970. See: + https://github.com/googleapis/python-bigquery-pandas/issues/365 + """ + import db_dtypes + import pandas.api.types + + # If you update this mapping, also update the table at + # `docs/reading.rst`. + dtype_map = { + "DATE": db_dtypes.DateDtype(), + "DATETIME": "datetime64[ns]", + "TIMESTAMP": "datetime64[ns]", + } + + for field in schema_fields: + # This method doesn't modify ARRAY/REPEATED columns. + if field["mode"].upper() == "REPEATED": + continue + + name = str(field["name"]) + dtype = dtype_map.get(field["type"].upper()) + + # Avoid deprecated conversion to timezone-naive dtype by only casting + # object dtypes. + if dtype and pandas.api.types.is_object_dtype(df[name]): + df[name] = df[name].astype(dtype, errors="ignore") + + # Ensure any TIMESTAMP columns are tz-aware. + df = pandas_gbq.timestamp.localize_df(df, schema_fields) + + return df + + +def _get_client(user_agent, rfc9110_delimiter, project_id, credentials): + import google.api_core.client_info + + bigquery = FEATURES.bigquery_try_import() + + user_agent = create_user_agent( + user_agent=user_agent, rfc9110_delimiter=rfc9110_delimiter + ) + + client_info = google.api_core.client_info.ClientInfo( + user_agent=user_agent, + ) + return bigquery.Client( + project=project_id, + credentials=credentials, + client_info=client_info, + ) + + +def create_user_agent( + user_agent: Optional[str] = None, rfc9110_delimiter: bool = False +) -> str: + """Creates a user agent string. + + The legacy format of our the user agent string was: `product-x.y.z` (where x, + y, and z are the major, minor, and micro version numbers). + + Users are able to prepend this string with their own user agent identifier + to render something similar to ` pandas-x.y.z`. + + The legacy format used a hyphen to separate the product from the product + version which differs slightly from the format recommended by RFC9110, which is: + `product/x.y.z`. To produce a user agent more in line with the RFC, set + rfc9110_delimiter to True. This setting does not depend on whether a + user_agent is also supplied. + + Reference: + https://www.rfc-editor.org/info/rfc9110 + + Args: + user_agent (Optional[str]): User agent string. + + rfc9110_delimiter (Optional[bool]): Sets delimiter to a hyphen or a slash. + Default is False, meaning a hyphen will be used. + + Returns (str): + Customized user agent string. + + Deprecation Warning: + In a future major release, the default delimiter will be changed to + a `/` in accordance with RFC9110. + """ + import pandas as pd + + if rfc9110_delimiter: + delimiter = "/" + else: + warnings.warn( + "In a future major release, the default delimiter will be " + "changed to a `/` in accordance with RFC9110.", + PendingDeprecationWarning, + stacklevel=2, + ) + delimiter = "-" + + identity = f"pandas{delimiter}{pd.__version__}" + + if user_agent is None: + user_agent = identity + else: + user_agent = f"{user_agent} {identity}" + + return user_agent diff --git a/packages/pandas-gbq/tests/system/test_read_gbq.py b/packages/pandas-gbq/tests/system/test_read_gbq.py index 72cb6b66ea6a..946da668ba81 100644 --- a/packages/pandas-gbq/tests/system/test_read_gbq.py +++ b/packages/pandas-gbq/tests/system/test_read_gbq.py @@ -645,7 +645,9 @@ def test_empty_dataframe(read_gbq, use_bqstorage_api): } ) result = read_gbq(query, use_bqstorage_api=use_bqstorage_api) - pandas.testing.assert_frame_equal(result, expected, check_index_type=False) + pandas.testing.assert_frame_equal( + result, expected, check_index_type=False, check_dtype=False + ) def test_dml_query(read_gbq, writable_table: str): From 9523cfa6dd032c96a85008211dee4ec1d6302d10 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 12 May 2025 14:53:14 -0700 Subject: [PATCH 484/519] chore(main): release 0.28.1 (#892) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- packages/pandas-gbq/CHANGELOG.md | 9 +++++++++ packages/pandas-gbq/pandas_gbq/version.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index a860458b1f13..b6d41d176d1d 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## [0.28.1](https://github.com/googleapis/python-bigquery-pandas/compare/v0.28.0...v0.28.1) (2025-04-28) + + +### Bug Fixes + +* Remove setup.cfg configuration for creating universal wheels ([#898](https://github.com/googleapis/python-bigquery-pandas/issues/898)) ([b3a9af2](https://github.com/googleapis/python-bigquery-pandas/commit/b3a9af27960c6e2736835d6a94e116f550f94114)) +* Remove unnecessary global variable definition ([#907](https://github.com/googleapis/python-bigquery-pandas/issues/907)) ([d4d85ce](https://github.com/googleapis/python-bigquery-pandas/commit/d4d85ce133fcdb98b53684f8dfca9d25c032b09e)) +* Resolve issue where pre-release versions of dependencies are installed ([#890](https://github.com/googleapis/python-bigquery-pandas/issues/890)) ([3574ca1](https://github.com/googleapis/python-bigquery-pandas/commit/3574ca181fe91ee65114835fd7db8bdd75fe135a)) + ## [0.28.0](https://github.com/googleapis/python-bigquery-pandas/compare/v0.27.0...v0.28.0) (2025-02-24) diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py index a6070398412b..5683780f5bc7 100644 --- a/packages/pandas-gbq/pandas_gbq/version.py +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -2,4 +2,4 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -__version__ = "0.28.0" +__version__ = "0.28.1" From c9720d13a7dbb259581a1aa50d9de210b3feab10 Mon Sep 17 00:00:00 2001 From: Shobhit Singh Date: Wed, 14 May 2025 12:21:36 -0700 Subject: [PATCH 485/519] feat: instrument vscode, jupyter and 3p plugin usage (#925) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: instrument vscode, jupyter and 3p plugin usage * fix the return value * add unit test coverage * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * use pathlib, reduce indentation, format --------- Co-authored-by: Owl Bot --- packages/pandas-gbq/pandas_gbq/environment.py | 99 +++++++++++++++++++ .../pandas-gbq/pandas_gbq/gbq_connector.py | 18 ++-- packages/pandas-gbq/tests/unit/test_to_gbq.py | 69 +++++++++++++ 3 files changed, 180 insertions(+), 6 deletions(-) create mode 100644 packages/pandas-gbq/pandas_gbq/environment.py diff --git a/packages/pandas-gbq/pandas_gbq/environment.py b/packages/pandas-gbq/pandas_gbq/environment.py new file mode 100644 index 000000000000..bf2c6d76bb77 --- /dev/null +++ b/packages/pandas-gbq/pandas_gbq/environment.py @@ -0,0 +1,99 @@ +# Copyright (c) 2025 pandas-gbq Authors All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + + +import importlib +import json +import os +import pathlib + +Path = pathlib.Path + + +# The identifier for GCP VS Code extension +# https://cloud.google.com/code/docs/vscode/install +GOOGLE_CLOUD_CODE_EXTENSION_NAME = "googlecloudtools.cloudcode" + + +# The identifier for BigQuery Jupyter notebook plugin +# https://cloud.google.com/bigquery/docs/jupyterlab-plugin +BIGQUERY_JUPYTER_PLUGIN_NAME = "bigquery_jupyter_plugin" + + +def _is_vscode_extension_installed(extension_id: str) -> bool: + """ + Checks if a given Visual Studio Code extension is installed. + + Args: + extension_id: The ID of the extension (e.g., "ms-python.python"). + + Returns: + True if the extension is installed, False otherwise. + """ + try: + # Determine the user's VS Code extensions directory. + user_home = Path.home() + vscode_extensions_dir = user_home / ".vscode" / "extensions" + + # Check if the extensions directory exists. + if not vscode_extensions_dir.exists(): + return False + + # Iterate through the subdirectories in the extensions directory. + for item in vscode_extensions_dir.iterdir(): + # Ignore non-directories. + if not item.is_dir(): + continue + + # Directory must start with the extension ID. + if not item.name.startswith(extension_id + "-"): + continue + + # As a more robust check, the manifest file must exist. + manifest_path = item / "package.json" + if not manifest_path.exists() or not manifest_path.is_file(): + continue + + # Finally, the manifest file must be a valid json + with open(manifest_path, "r", encoding="utf-8") as f: + json.load(f) + + return True + except Exception: + pass + + return False + + +def _is_package_installed(package_name: str) -> bool: + """ + Checks if a Python package is installed. + + Args: + package_name: The name of the package to check (e.g., "requests", "numpy"). + + Returns: + True if the package is installed, False otherwise. + """ + try: + importlib.import_module(package_name) + return True + except Exception: + return False + + +def is_vscode() -> bool: + return os.getenv("VSCODE_PID") is not None + + +def is_jupyter() -> bool: + return os.getenv("JPY_PARENT_PID") is not None + + +def is_vscode_google_cloud_code_extension_installed() -> bool: + return _is_vscode_extension_installed(GOOGLE_CLOUD_CODE_EXTENSION_NAME) + + +def is_jupyter_bigquery_plugin_installed() -> bool: + return _is_package_installed(BIGQUERY_JUPYTER_PLUGIN_NAME) diff --git a/packages/pandas-gbq/pandas_gbq/gbq_connector.py b/packages/pandas-gbq/pandas_gbq/gbq_connector.py index 97a22db4a8d7..2b3b716e8b99 100644 --- a/packages/pandas-gbq/pandas_gbq/gbq_connector.py +++ b/packages/pandas-gbq/pandas_gbq/gbq_connector.py @@ -19,6 +19,7 @@ import pandas_gbq.constants from pandas_gbq.contexts import context +import pandas_gbq.environment as environment import pandas_gbq.exceptions from pandas_gbq.exceptions import ( GenericGBQException, @@ -517,11 +518,16 @@ def create_user_agent( ) delimiter = "-" - identity = f"pandas{delimiter}{pd.__version__}" + identities = [] if user_agent is None else [user_agent] + identities.append(f"pandas{delimiter}{pd.__version__}") - if user_agent is None: - user_agent = identity - else: - user_agent = f"{user_agent} {identity}" + if environment.is_vscode(): + identities.append("vscode") + if environment.is_vscode_google_cloud_code_extension_installed(): + identities.append(environment.GOOGLE_CLOUD_CODE_EXTENSION_NAME) + elif environment.is_jupyter(): + identities.append("jupyter") + if environment.is_jupyter_bigquery_plugin_installed(): + identities.append(environment.BIGQUERY_JUPYTER_PLUGIN_NAME) - return user_agent + return " ".join(identities) diff --git a/packages/pandas-gbq/tests/unit/test_to_gbq.py b/packages/pandas-gbq/tests/unit/test_to_gbq.py index f4012dc84353..083673c7f97a 100644 --- a/packages/pandas-gbq/tests/unit/test_to_gbq.py +++ b/packages/pandas-gbq/tests/unit/test_to_gbq.py @@ -2,6 +2,12 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. + +import os +import pathlib +import tempfile +import unittest.mock as mock + import google.api_core.exceptions import google.cloud.bigquery import pandas as pd @@ -10,6 +16,8 @@ from pandas_gbq import gbq +Path = pathlib.Path + class FakeDataFrame: """A fake bigframes DataFrame to avoid depending on bigframes.""" @@ -202,3 +210,64 @@ def test_create_user_agent(user_agent, rfc9110_delimiter, expected): result = create_user_agent(user_agent, rfc9110_delimiter) assert result == expected + + +@mock.patch.dict(os.environ, {"VSCODE_PID": "1234"}, clear=True) +def test_create_user_agent_vscode(): + from pandas_gbq.gbq import create_user_agent + + assert create_user_agent() == f"pandas-{pd.__version__} vscode" + + +@mock.patch.dict(os.environ, {"VSCODE_PID": "1234"}, clear=True) +def test_create_user_agent_vscode_plugin(): + from pandas_gbq.gbq import create_user_agent + + with tempfile.TemporaryDirectory() as tmpdir: + user_home = Path(tmpdir) + plugin_dir = ( + user_home / ".vscode" / "extensions" / "googlecloudtools.cloudcode-0.12" + ) + plugin_config = plugin_dir / "package.json" + + # originally pluging config does not exist + assert not plugin_config.exists() + + # simulate plugin installation by creating plugin config on disk + plugin_dir.mkdir(parents=True) + with open(plugin_config, "w") as f: + f.write("{}") + + with mock.patch("pathlib.Path.home", return_value=user_home): + assert ( + create_user_agent() + == f"pandas-{pd.__version__} vscode googlecloudtools.cloudcode" + ) + + +@mock.patch.dict(os.environ, {"JPY_PARENT_PID": "1234"}, clear=True) +def test_create_user_agent_jupyter(): + from pandas_gbq.gbq import create_user_agent + + assert create_user_agent() == f"pandas-{pd.__version__} jupyter" + + +@mock.patch.dict(os.environ, {"JPY_PARENT_PID": "1234"}, clear=True) +def test_create_user_agent_jupyter_extension(): + from pandas_gbq.gbq import create_user_agent + + def custom_import_module_side_effect(name, package=None): + if name == "bigquery_jupyter_plugin": + return mock.MagicMock() + else: + import importlib + + return importlib.import_module(name, package) + + with mock.patch( + "importlib.import_module", side_effect=custom_import_module_side_effect + ): + assert ( + create_user_agent() + == f"pandas-{pd.__version__} jupyter bigquery_jupyter_plugin" + ) From e40e7ae416d9a381b177fea839d11b7313372a15 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 19 May 2025 10:18:11 -0700 Subject: [PATCH 486/519] chore(main): release 0.29.0 (#926) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- packages/pandas-gbq/CHANGELOG.md | 7 +++++++ packages/pandas-gbq/pandas_gbq/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index b6d41d176d1d..b16acdab7083 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.29.0](https://github.com/googleapis/python-bigquery-pandas/compare/v0.28.1...v0.29.0) (2025-05-14) + + +### Features + +* Instrument vscode, jupyter and 3p plugin usage ([#925](https://github.com/googleapis/python-bigquery-pandas/issues/925)) ([7d354f1](https://github.com/googleapis/python-bigquery-pandas/commit/7d354f1ca90eb2647c81d8d8422d0146d56f802a)) + ## [0.28.1](https://github.com/googleapis/python-bigquery-pandas/compare/v0.28.0...v0.28.1) (2025-04-28) diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py index 5683780f5bc7..e9724daf41c4 100644 --- a/packages/pandas-gbq/pandas_gbq/version.py +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -2,4 +2,4 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -__version__ = "0.28.1" +__version__ = "0.29.0" From aca5c88678c3afb79e10bf4acba8a835d08015b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Swe=C3=B1a=20=28Swast=29?= Date: Tue, 3 Jun 2025 14:17:37 -0500 Subject: [PATCH 487/519] fix: remove pandas-gbq client ID for authentication (#927) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: remove pandas-gbq client ID for authentication pandas-gbq's client ID has been failing for some time due to `Error 400: redirect_uri_mismatch`. Since it's only used for a fallback if no application default credentials are available, it's probably OK to use pydata-google-auth's client ID instead. * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot --- packages/pandas-gbq/pandas_gbq/auth.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/packages/pandas-gbq/pandas_gbq/auth.py b/packages/pandas-gbq/pandas_gbq/auth.py index 0dbc2a07414b..704f15be1c0c 100644 --- a/packages/pandas-gbq/pandas_gbq/auth.py +++ b/packages/pandas-gbq/pandas_gbq/auth.py @@ -13,19 +13,6 @@ CREDENTIALS_CACHE_FILENAME = "bigquery_credentials.dat" SCOPES = ["https://www.googleapis.com/auth/bigquery"] -# The following constants are used for end-user authentication. -# It identifies (via credentials from the pandas-gbq-auth GCP project) the -# application that is requesting permission to access the BigQuery API on -# behalf of a G Suite or Gmail user. -# -# In a web application, the client secret would be kept secret, but this is not -# possible for applications that are installed locally on an end-user's -# machine. -# -# See: https://cloud.google.com/docs/authentication/end-user for details. -CLIENT_ID = "725825577420-unm2gnkiprugilg743tkbig250f4sfsj.apps.googleusercontent.com" -CLIENT_SECRET = "4hqze9yI8fxShls8eJWkeMdJ" - def get_credentials( private_key=None, @@ -47,12 +34,6 @@ def get_credentials( method from the google-auth package.""" ) - if client_id is None: - client_id = CLIENT_ID - - if client_secret is None: - client_secret = CLIENT_SECRET - credentials, default_project_id = pydata_google_auth.default( SCOPES, client_id=client_id, From f580e5317bc4c96a2b78ab5d074f1a3612b46f1b Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 9 Jun 2025 09:18:25 -0500 Subject: [PATCH 488/519] chore(main): release 0.29.1 (#928) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- packages/pandas-gbq/CHANGELOG.md | 7 +++++++ packages/pandas-gbq/pandas_gbq/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index b16acdab7083..5e60c5bb17ec 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.29.1](https://github.com/googleapis/python-bigquery-pandas/compare/v0.29.0...v0.29.1) (2025-06-03) + + +### Bug Fixes + +* Remove pandas-gbq client ID for authentication ([#927](https://github.com/googleapis/python-bigquery-pandas/issues/927)) ([8bb7401](https://github.com/googleapis/python-bigquery-pandas/commit/8bb74015ca47ccca37cb5077bae265a257bec8cc)) + ## [0.29.0](https://github.com/googleapis/python-bigquery-pandas/compare/v0.28.1...v0.29.0) (2025-05-14) diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py index e9724daf41c4..90bd1ac47d6c 100644 --- a/packages/pandas-gbq/pandas_gbq/version.py +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -2,4 +2,4 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -__version__ = "0.29.0" +__version__ = "0.29.1" From 47d1a0523da87edc7439202daa6da6ee4f6f536d Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Wed, 18 Jun 2025 10:52:41 -0400 Subject: [PATCH 489/519] feat: Add Python 3.13 support (#930) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add Python 3.13 support This commit introduces support for Python 3.13 as a runtime dependency. The following changes were made: - Updated `noxfile.py` to include Python 3.13 in unit and system test versions. - Created `testing/constraints-3.13.txt` (initially empty, copied from an empty constraints-3.12.txt). - Updated `.github/workflows/unittest.yml` to include Python 3.13 in the test matrix. - Updated `setup.py` to add the Python 3.13 classifier. - Updated `CONTRIBUTING.rst` to list Python 3.13 as a supported version. - Created `.kokoro/presubmit/system-3.13.cfg` for Kokoro system tests. - Updated `.github/sync-repo-settings.yaml` to include Python 3.13 in required status checks. - Updated `owlbot.py` to include Python 3.13 in unit and system test versions for templated files. * feat: Add Python 3.13 support This commit introduces support for Python 3.13 as a runtime dependency. The following changes were made: - Updated `noxfile.py` to include Python 3.13 in unit and system test versions. - Created `testing/constraints-3.13.txt` (initially empty, copied from an empty constraints-3.12.txt). - Updated `.github/workflows/unittest.yml` to include Python 3.13 in the test matrix. - Updated `setup.py` to add the Python 3.13 classifier. - Updated `CONTRIBUTING.rst` to list Python 3.13 as a supported version. - Created `.kokoro/presubmit/system-3.13.cfg` for Kokoro system tests. - Updated `.github/sync-repo-settings.yaml` to include Python 3.13 in required status checks. - Updated `owlbot.py` to include Python 3.13 in unit and system test versions for templated files. * removes presubmit task for 3.12 now that we have one for 3.13 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * Update CONTRIBUTING.rst * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: google-labs-jules[bot] <161369871+google-labs-jules[bot]@users.noreply.github.com> Co-authored-by: Owl Bot --- packages/pandas-gbq/.github/sync-repo-settings.yaml | 2 ++ packages/pandas-gbq/.github/workflows/unittest.yml | 2 +- .../presubmit/{system-3.12.cfg => system-3.13.cfg} | 4 ++-- packages/pandas-gbq/CONTRIBUTING.rst | 10 ++++++---- packages/pandas-gbq/noxfile.py | 4 ++-- packages/pandas-gbq/owlbot.py | 4 ++-- packages/pandas-gbq/setup.py | 1 + packages/pandas-gbq/testing/constraints-3.13.txt | 0 8 files changed, 16 insertions(+), 11 deletions(-) rename packages/pandas-gbq/.kokoro/presubmit/{system-3.12.cfg => system-3.13.cfg} (83%) create mode 100644 packages/pandas-gbq/testing/constraints-3.13.txt diff --git a/packages/pandas-gbq/.github/sync-repo-settings.yaml b/packages/pandas-gbq/.github/sync-repo-settings.yaml index a570d4affa97..692ec5b69655 100644 --- a/packages/pandas-gbq/.github/sync-repo-settings.yaml +++ b/packages/pandas-gbq/.github/sync-repo-settings.yaml @@ -16,6 +16,7 @@ branchProtectionRules: - 'unit (3.10)' - 'unit (3.11)' - 'unit (3.12)' + - 'unit (3.13)' - 'cover' - 'Kokoro' - 'Samples - Lint' @@ -24,6 +25,7 @@ branchProtectionRules: - 'Samples - Python 3.10' - 'Samples - Python 3.11' - 'Samples - Python 3.12' + - 'Samples - Python 3.13' permissionRules: - team: actools-python permission: admin diff --git a/packages/pandas-gbq/.github/workflows/unittest.yml b/packages/pandas-gbq/.github/workflows/unittest.yml index 107eac6b4b61..4a4c5d93bc8b 100644 --- a/packages/pandas-gbq/.github/workflows/unittest.yml +++ b/packages/pandas-gbq/.github/workflows/unittest.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - python: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] steps: - name: Checkout uses: actions/checkout@v4 diff --git a/packages/pandas-gbq/.kokoro/presubmit/system-3.12.cfg b/packages/pandas-gbq/.kokoro/presubmit/system-3.13.cfg similarity index 83% rename from packages/pandas-gbq/.kokoro/presubmit/system-3.12.cfg rename to packages/pandas-gbq/.kokoro/presubmit/system-3.13.cfg index 28bbbe4c25b7..3ec53cf9a70c 100644 --- a/packages/pandas-gbq/.kokoro/presubmit/system-3.12.cfg +++ b/packages/pandas-gbq/.kokoro/presubmit/system-3.13.cfg @@ -3,5 +3,5 @@ # Only run the following session(s) env_vars: { key: "NOX_SESSION" - value: "system-3.12" -} \ No newline at end of file + value: "system-3.13" +} diff --git a/packages/pandas-gbq/CONTRIBUTING.rst b/packages/pandas-gbq/CONTRIBUTING.rst index 620763e35694..28f54669c078 100644 --- a/packages/pandas-gbq/CONTRIBUTING.rst +++ b/packages/pandas-gbq/CONTRIBUTING.rst @@ -22,7 +22,7 @@ In order to add a feature: documentation. - The feature must work fully on the following CPython versions: - 3.8, 3.9, 3.10, 3.11 and 3.12 on both UNIX and Windows. + 3.8, 3.9, 3.10, 3.11, 3.12 and 3.13 on both UNIX and Windows. - The feature must not add unnecessary dependencies (where "unnecessary" is of course subjective, but new dependencies should @@ -72,7 +72,7 @@ We use `nox `__ to instrument our tests. - To run a single unit test:: - $ nox -s unit-3.12 -- -k + $ nox -s unit-3.13 -- -k .. note:: @@ -143,12 +143,12 @@ Running System Tests $ nox -s system # Run a single system test - $ nox -s system-3.12 -- -k + $ nox -s system-3.13 -- -k .. note:: - System tests are only configured to run under Python 3.8, 3.9, 3.10, 3.11 and 3.12. + System tests are only configured to run under Python 3.8, 3.9, 3.10, 3.11, 3.12 and 3.13. For expediency, we do not run them in older versions of Python 3. This alone will not run the tests. You'll need to change some local @@ -226,12 +226,14 @@ We support: - `Python 3.10`_ - `Python 3.11`_ - `Python 3.12`_ +- `Python 3.13`_ .. _Python 3.8: https://docs.python.org/3.8/ .. _Python 3.9: https://docs.python.org/3.9/ .. _Python 3.10: https://docs.python.org/3.10/ .. _Python 3.11: https://docs.python.org/3.11/ .. _Python 3.12: https://docs.python.org/3.12/ +.. _Python 3.13: https://docs.python.org/3.13/ Supported versions can be found in our ``noxfile.py`` `config`_. diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 33923771e630..52bdcde16c00 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -35,7 +35,7 @@ DEFAULT_PYTHON_VERSION = "3.10" -UNIT_TEST_PYTHON_VERSIONS = ["3.8", "3.9", "3.10", "3.11", "3.12"] +UNIT_TEST_PYTHON_VERSIONS = ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] UNIT_TEST_STANDARD_DEPENDENCIES = [ "mock", "asyncmock", @@ -57,7 +57,7 @@ "3.9": [], } -SYSTEM_TEST_PYTHON_VERSIONS = ["3.8", "3.9", "3.10", "3.11", "3.12"] +SYSTEM_TEST_PYTHON_VERSIONS = ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] SYSTEM_TEST_STANDARD_DEPENDENCIES = [ "mock", "pytest", diff --git a/packages/pandas-gbq/owlbot.py b/packages/pandas-gbq/owlbot.py index 96a795c3f319..35e19fcfb7b7 100644 --- a/packages/pandas-gbq/owlbot.py +++ b/packages/pandas-gbq/owlbot.py @@ -35,8 +35,8 @@ extras = ["tqdm", "geopandas"] templated_files = common.py_library( default_python_version="3.10", - unit_test_python_versions=["3.8", "3.9", "3.10", "3.11", "3.12"], - system_test_python_versions=["3.8", "3.9", "3.10", "3.11", "3.12"], + unit_test_python_versions=["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"], + system_test_python_versions=["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"], cov_level=96, unit_test_external_dependencies=["freezegun"], unit_test_extras=extras, diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index 6f84ef68780d..681a80aaf067 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -90,6 +90,7 @@ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Operating System :: OS Independent", "Topic :: Internet", "Topic :: Scientific/Engineering", diff --git a/packages/pandas-gbq/testing/constraints-3.13.txt b/packages/pandas-gbq/testing/constraints-3.13.txt new file mode 100644 index 000000000000..e69de29bb2d1 From 62651f980e14adae7186de086b8bcc73975625d9 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Wed, 25 Jun 2025 07:43:24 -0400 Subject: [PATCH 490/519] deps!: Remove support for Python 3.8 (#932) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Remove support for Python 3.8 This commit removes Python 3.8 from the supported versions. Updates include changes in noxfiles, GitHub workflows, setup.py, Kokoro configurations, and documentation to reflect Python 3.9 as the minimum supported version. * Apply follow-up changes for Python 3.8 removal This commit addresses items missed in the initial Python 3.8 removal: * Adds `kokoro/presubmit/system-3.9.cfg`. * Updates example commands in `CONTRIBUTING.rst`. * Modifies the warning in `pandas_gbq/__init__.py` for Python < 3.9. * Updates Python versions in `owlbot.py`. * Removes 3.8-specific line from `samples/snippets/requirements.txt`. * Populates `testing/constraints-3.9.txt` with correct lower bounds. * Update pandas_gbq/__init__.py * Update samples/snippets/noxfile.py * Update requirements.txt * Update testing/constraints-3.9.txt * Update owlbot.py * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: google-labs-jules[bot] <161369871+google-labs-jules[bot]@users.noreply.github.com> Co-authored-by: Owl Bot --- .../.github/sync-repo-settings.yaml | 2 -- .../pandas-gbq/.github/workflows/unittest.yml | 2 +- .../{system-3.8.cfg => system-3.9.cfg} | 4 ++-- packages/pandas-gbq/CONTRIBUTING.rst | 8 +++---- packages/pandas-gbq/noxfile.py | 4 ++-- packages/pandas-gbq/owlbot.py | 7 +++--- packages/pandas-gbq/pandas_gbq/__init__.py | 14 ++++++------ .../samples/snippets/requirements.txt | 5 ++--- packages/pandas-gbq/setup.py | 3 +-- .../pandas-gbq/testing/constraints-3.8.txt | 22 ------------------- .../pandas-gbq/testing/constraints-3.9.txt | 20 +++++++++++++++++ 11 files changed, 42 insertions(+), 49 deletions(-) rename packages/pandas-gbq/.kokoro/presubmit/{system-3.8.cfg => system-3.9.cfg} (83%) delete mode 100644 packages/pandas-gbq/testing/constraints-3.8.txt diff --git a/packages/pandas-gbq/.github/sync-repo-settings.yaml b/packages/pandas-gbq/.github/sync-repo-settings.yaml index 692ec5b69655..130dd1fef4dc 100644 --- a/packages/pandas-gbq/.github/sync-repo-settings.yaml +++ b/packages/pandas-gbq/.github/sync-repo-settings.yaml @@ -11,7 +11,6 @@ branchProtectionRules: - 'OwlBot Post Processor' - 'docs' - 'lint' - - 'unit (3.8)' - 'unit (3.9)' - 'unit (3.10)' - 'unit (3.11)' @@ -20,7 +19,6 @@ branchProtectionRules: - 'cover' - 'Kokoro' - 'Samples - Lint' - - 'Samples - Python 3.8' - 'Samples - Python 3.9' - 'Samples - Python 3.10' - 'Samples - Python 3.11' diff --git a/packages/pandas-gbq/.github/workflows/unittest.yml b/packages/pandas-gbq/.github/workflows/unittest.yml index 4a4c5d93bc8b..7137d0ad257f 100644 --- a/packages/pandas-gbq/.github/workflows/unittest.yml +++ b/packages/pandas-gbq/.github/workflows/unittest.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - python: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] + python: ['3.9', '3.10', '3.11', '3.12', '3.13'] steps: - name: Checkout uses: actions/checkout@v4 diff --git a/packages/pandas-gbq/.kokoro/presubmit/system-3.8.cfg b/packages/pandas-gbq/.kokoro/presubmit/system-3.9.cfg similarity index 83% rename from packages/pandas-gbq/.kokoro/presubmit/system-3.8.cfg rename to packages/pandas-gbq/.kokoro/presubmit/system-3.9.cfg index 15b14528baf7..be5a81249af7 100644 --- a/packages/pandas-gbq/.kokoro/presubmit/system-3.8.cfg +++ b/packages/pandas-gbq/.kokoro/presubmit/system-3.9.cfg @@ -3,5 +3,5 @@ # Only run the following session(s) env_vars: { key: "NOX_SESSION" - value: "system-3.8" -} \ No newline at end of file + value: "system-3.9" +} diff --git a/packages/pandas-gbq/CONTRIBUTING.rst b/packages/pandas-gbq/CONTRIBUTING.rst index 28f54669c078..2e8e9860f4f5 100644 --- a/packages/pandas-gbq/CONTRIBUTING.rst +++ b/packages/pandas-gbq/CONTRIBUTING.rst @@ -22,7 +22,7 @@ In order to add a feature: documentation. - The feature must work fully on the following CPython versions: - 3.8, 3.9, 3.10, 3.11, 3.12 and 3.13 on both UNIX and Windows. + 3.9, 3.10, 3.11, 3.12 and 3.13 on both UNIX and Windows. - The feature must not add unnecessary dependencies (where "unnecessary" is of course subjective, but new dependencies should @@ -148,7 +148,7 @@ Running System Tests .. note:: - System tests are only configured to run under Python 3.8, 3.9, 3.10, 3.11, 3.12 and 3.13. + System tests are only configured to run under Python 3.9, 3.10, 3.11, 3.12 and 3.13. For expediency, we do not run them in older versions of Python 3. This alone will not run the tests. You'll need to change some local @@ -221,14 +221,12 @@ Supported Python Versions We support: -- `Python 3.8`_ - `Python 3.9`_ - `Python 3.10`_ - `Python 3.11`_ - `Python 3.12`_ - `Python 3.13`_ -.. _Python 3.8: https://docs.python.org/3.8/ .. _Python 3.9: https://docs.python.org/3.9/ .. _Python 3.10: https://docs.python.org/3.10/ .. _Python 3.11: https://docs.python.org/3.11/ @@ -241,7 +239,7 @@ Supported versions can be found in our ``noxfile.py`` `config`_. .. _config: https://github.com/googleapis/python-bigquery-pandas/blob/main/noxfile.py -We also explicitly decided to support Python 3 beginning with version 3.8. +We also explicitly decided to support Python 3 beginning with version 3.9. Reasons for this include: - Encouraging use of newest versions of Python 3 diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 52bdcde16c00..e246b05d1eda 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -35,7 +35,7 @@ DEFAULT_PYTHON_VERSION = "3.10" -UNIT_TEST_PYTHON_VERSIONS = ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] +UNIT_TEST_PYTHON_VERSIONS = ["3.9", "3.10", "3.11", "3.12", "3.13"] UNIT_TEST_STANDARD_DEPENDENCIES = [ "mock", "asyncmock", @@ -57,7 +57,7 @@ "3.9": [], } -SYSTEM_TEST_PYTHON_VERSIONS = ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] +SYSTEM_TEST_PYTHON_VERSIONS = ["3.9", "3.10", "3.11", "3.12", "3.13"] SYSTEM_TEST_STANDARD_DEPENDENCIES = [ "mock", "pytest", diff --git a/packages/pandas-gbq/owlbot.py b/packages/pandas-gbq/owlbot.py index 35e19fcfb7b7..1d5d912f3e56 100644 --- a/packages/pandas-gbq/owlbot.py +++ b/packages/pandas-gbq/owlbot.py @@ -35,8 +35,8 @@ extras = ["tqdm", "geopandas"] templated_files = common.py_library( default_python_version="3.10", - unit_test_python_versions=["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"], - system_test_python_versions=["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"], + unit_test_python_versions=["3.9", "3.10", "3.11", "3.12", "3.13"], + system_test_python_versions=["3.9", "3.10", "3.11", "3.12", "3.13"], cov_level=96, unit_test_external_dependencies=["freezegun"], unit_test_extras=extras, @@ -57,8 +57,9 @@ "docs/multiprocessing.rst", "noxfile.py", "README.rst", - # exclude this file as we have an alternate prerelease.cfg ".github/workflows/docs.yml", + ".github/sync-repo-settings.yaml", + # exclude this file as we have an alternate prerelease.cfg ".kokoro/presubmit/prerelease-deps.cfg", ".kokoro/presubmit/presubmit.cfg", ], diff --git a/packages/pandas-gbq/pandas_gbq/__init__.py b/packages/pandas-gbq/pandas_gbq/__init__.py index 184f8c4416f7..a842c81f5b0c 100644 --- a/packages/pandas-gbq/pandas_gbq/__init__.py +++ b/packages/pandas-gbq/pandas_gbq/__init__.py @@ -11,14 +11,14 @@ from .gbq import read_gbq, to_gbq # noqa sys_major, sys_minor, sys_micro = _versions_helpers.extract_runtime_version() -if sys_major == 3 and sys_minor in (7, 8): +if sys_major == 3 and sys_minor < 9: warnings.warn( - "The python-bigquery library will stop supporting Python 3.7 " - "and Python 3.8 in a future major release expected in Q4 2024. " - f"Your Python version is {sys_major}.{sys_minor}.{sys_micro}. We " - "recommend that you update soon to ensure ongoing support. For " - "more details, see: [Google Cloud Client Libraries Supported Python Versions policy](https://cloud.google.com/python/docs/supported-python-versions)", - PendingDeprecationWarning, + "pandas-gbq no longer supports Python versions older than 3.9. " + "Your Python version is " + f"{sys_major}.{sys_minor}.{sys_micro}. Please update " + "to Python 3.9 or newer to ensure ongoing support. For more details, " + "see: https://cloud.google.com/python/docs/supported-python-versions", + FutureWarning, ) __version__ = pandas_gbq_version.__version__ diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index a451f10ac5b3..ca1e1d0c2d34 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,6 +1,5 @@ google-cloud-bigquery-storage==2.29.1 google-cloud-bigquery==3.30.0 pandas-gbq==0.28.0 -pandas===2.0.3; python_version == '3.8' -pandas==2.2.3; python_version >= '3.9' -pyarrow==19.0.1; python_version >= '3.9' +pandas==2.2.3 +pyarrow==19.0.1 diff --git a/packages/pandas-gbq/setup.py b/packages/pandas-gbq/setup.py index 681a80aaf067..893d801b6460 100644 --- a/packages/pandas-gbq/setup.py +++ b/packages/pandas-gbq/setup.py @@ -85,7 +85,6 @@ "License :: OSI Approved :: BSD License", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", @@ -99,7 +98,7 @@ packages=packages, install_requires=dependencies, extras_require=extras, - python_requires=">=3.8", + python_requires=">=3.9", include_package_data=True, zip_safe=False, ) diff --git a/packages/pandas-gbq/testing/constraints-3.8.txt b/packages/pandas-gbq/testing/constraints-3.8.txt deleted file mode 100644 index 8d6ef4f493db..000000000000 --- a/packages/pandas-gbq/testing/constraints-3.8.txt +++ /dev/null @@ -1,22 +0,0 @@ -# This constraints file is used to check that lower bounds -# are correct in setup.py -# List *all* library dependencies and extras in this file. -# Pin the version to the lower bound. -# -# e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", -# Then this file should have foo==1.14.0 -# protobuf==3.19.5 -db-dtypes==1.0.4 -geopandas==0.9.0 -google-api-core==2.10.2 -google-auth==2.13.0 -google-auth-oauthlib==0.7.0 -google-cloud-bigquery==3.4.2 -google-cloud-bigquery-storage==2.16.2 -numpy==1.18.1 -pandas==1.1.4 -pyarrow==4.0.0 -pydata-google-auth==1.5.0 -Shapely==1.8.4 -tqdm==4.23.0 -packaging==22.0.0 diff --git a/packages/pandas-gbq/testing/constraints-3.9.txt b/packages/pandas-gbq/testing/constraints-3.9.txt index 76864a661daf..db8a499a617d 100644 --- a/packages/pandas-gbq/testing/constraints-3.9.txt +++ b/packages/pandas-gbq/testing/constraints-3.9.txt @@ -1,2 +1,22 @@ +# This constraints file is used to check that lower bounds +# are correct in setup.py +# List *all* library dependencies and extras in this file. +# Pin the version to the lower bound. +# +# e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", +# Then this file should have foo==1.14.0 +db-dtypes==1.0.4 numpy==1.19.4 pandas==1.1.4 +pyarrow==4.0.0 +pydata-google-auth==1.5.0 +google-api-core==2.10.2 +google-auth==2.13.0 +google-auth-oauthlib==0.7.0 +google-cloud-bigquery==3.4.2 +packaging==22.0.0 +# Extras +google-cloud-bigquery-storage==2.16.2 +tqdm==4.23.0 +geopandas==0.9.0 +Shapely==1.8.4 From 12e507e493a4172655be82640850e4106c73f356 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Fri, 27 Jun 2025 13:13:35 -0400 Subject: [PATCH 491/519] chore(deps): update all dependencies (#933) * chore(deps): update all dependencies * updates some configs in owlbot, noxfile, and ymls to ensure that the correct version of python is used * Update noxfile.py * Update noxfile.py * Update noxfile.py --------- Co-authored-by: Mend Renovate --- packages/pandas-gbq/noxfile.py | 1 - packages/pandas-gbq/owlbot.py | 7 ++++++- packages/pandas-gbq/renovate.json | 2 +- .../pandas-gbq/samples/snippets/requirements-test.txt | 4 ++-- packages/pandas-gbq/samples/snippets/requirements.txt | 10 +++++----- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index e246b05d1eda..46ed30076734 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -34,7 +34,6 @@ DEFAULT_PYTHON_VERSION = "3.10" - UNIT_TEST_PYTHON_VERSIONS = ["3.9", "3.10", "3.11", "3.12", "3.13"] UNIT_TEST_STANDARD_DEPENDENCIES = [ "mock", diff --git a/packages/pandas-gbq/owlbot.py b/packages/pandas-gbq/owlbot.py index 1d5d912f3e56..cde35a98b10e 100644 --- a/packages/pandas-gbq/owlbot.py +++ b/packages/pandas-gbq/owlbot.py @@ -57,11 +57,16 @@ "docs/multiprocessing.rst", "noxfile.py", "README.rst", - ".github/workflows/docs.yml", + ".github/workflows/docs.yml", # to avoid overwriting python version + ".github/workflows/lint.yml", # to avoid overwriting python version ".github/sync-repo-settings.yaml", # exclude this file as we have an alternate prerelease.cfg ".kokoro/presubmit/prerelease-deps.cfg", ".kokoro/presubmit/presubmit.cfg", + "renovate.json", # to avoid overwriting the ignorePaths list additions: + # ".github/workflows/docs.yml AND lint.yml" specifically + # the version of python referenced in each of those files. + # Currently renovate bot wants to change 3.10 to 3.13. ], ) diff --git a/packages/pandas-gbq/renovate.json b/packages/pandas-gbq/renovate.json index c7875c469bd5..9d9a6d0bb8cc 100644 --- a/packages/pandas-gbq/renovate.json +++ b/packages/pandas-gbq/renovate.json @@ -5,7 +5,7 @@ ":preserveSemverRanges", ":disableDependencyDashboard" ], - "ignorePaths": [".pre-commit-config.yaml", ".kokoro/requirements.txt", "setup.py", ".github/workflows/unittest.yml"], + "ignorePaths": [".pre-commit-config.yaml", ".kokoro/requirements.txt", "setup.py", ".github/workflows/unittest.yml", ".github/workflows/docs.yml", ".github/workflows/lint.yml"], "pip_requirements": { "fileMatch": ["requirements-test.txt", "samples/[\\S/]*constraints.txt", "samples/[\\S/]*constraints-test.txt"] } diff --git a/packages/pandas-gbq/samples/snippets/requirements-test.txt b/packages/pandas-gbq/samples/snippets/requirements-test.txt index 417ef03776ef..2c7482f89a4b 100644 --- a/packages/pandas-gbq/samples/snippets/requirements-test.txt +++ b/packages/pandas-gbq/samples/snippets/requirements-test.txt @@ -1,2 +1,2 @@ -google-cloud-testutils==1.6.0 -pytest==8.3.5 +google-cloud-testutils==1.6.4 +pytest==8.4.1 diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index ca1e1d0c2d34..74d3b9cb0e13 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,5 +1,5 @@ -google-cloud-bigquery-storage==2.29.1 -google-cloud-bigquery==3.30.0 -pandas-gbq==0.28.0 -pandas==2.2.3 -pyarrow==19.0.1 +google-cloud-bigquery-storage==2.32.0 +google-cloud-bigquery==3.34.0 +pandas-gbq==0.29.1 +pandas==2.3.0 +pyarrow==20.0.0 From ed2abae66a008d2aae4f35fedad6acc656ad5fb7 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Thu, 10 Jul 2025 12:14:44 -0400 Subject: [PATCH 492/519] chore(main): release 0.29.2 (#938) * chore(main): release 0.29.2 * Update CHANGELOG.md --------- Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> Co-authored-by: Chalmer Lowe --- packages/pandas-gbq/CHANGELOG.md | 17 +++++++++++++++++ packages/pandas-gbq/pandas_gbq/version.py | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index 5e60c5bb17ec..1cd1478cbceb 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,22 @@ # Changelog +## [0.29.2](https://github.com/googleapis/python-bigquery-pandas/compare/v0.29.1...v0.29.2) (2025-07-10) + + +### Features + +* Add Python 3.13 support ([#930](https://github.com/googleapis/python-bigquery-pandas/issues/930)) ([76e6d11](https://github.com/googleapis/python-bigquery-pandas/commit/76e6d11aaf674bc4bb4de223845cc31fe3cf5765)) + + +### Dependencies + +* Remove support for Python 3.8 ([#932](https://github.com/googleapis/python-bigquery-pandas/issues/932)) ([ba35a9c](https://github.com/googleapis/python-bigquery-pandas/commit/ba35a9c3fe1acef13a629dacbc92d00f4291aa63)) + + +### Miscellaneous Chores + +* Release 0.29.2 ([#937](https://github.com/googleapis/python-bigquery-pandas/issues/937)) ([e595c2b](https://github.com/googleapis/python-bigquery-pandas/commit/e595c2b1b237252e48004fcb77c0f33ddbd42b5a)) + ## [0.29.1](https://github.com/googleapis/python-bigquery-pandas/compare/v0.29.0...v0.29.1) (2025-06-03) diff --git a/packages/pandas-gbq/pandas_gbq/version.py b/packages/pandas-gbq/pandas_gbq/version.py index 90bd1ac47d6c..3c11b41edb86 100644 --- a/packages/pandas-gbq/pandas_gbq/version.py +++ b/packages/pandas-gbq/pandas_gbq/version.py @@ -2,4 +2,4 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -__version__ = "0.29.1" +__version__ = "0.29.2" From 875ef1ef8a9068a5aadd283d1d646962052b6a9d Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Thu, 31 Jul 2025 23:48:31 +0200 Subject: [PATCH 493/519] chore(deps): update all dependencies (#936) --- packages/pandas-gbq/samples/snippets/requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 74d3b9cb0e13..1c77bf37edec 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,5 +1,5 @@ google-cloud-bigquery-storage==2.32.0 -google-cloud-bigquery==3.34.0 -pandas-gbq==0.29.1 -pandas==2.3.0 -pyarrow==20.0.0 +google-cloud-bigquery==3.35.1 +pandas-gbq==0.29.2 +pandas==2.3.1 +pyarrow==21.0.0 From 418b630c356bf610a40101c7f6b4d6657fc96dd6 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Wed, 3 Sep 2025 18:18:18 +0200 Subject: [PATCH 494/519] chore(deps): update all dependencies (#946) --- packages/pandas-gbq/samples/snippets/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 1c77bf37edec..51d8b62ba524 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,5 +1,5 @@ google-cloud-bigquery-storage==2.32.0 -google-cloud-bigquery==3.35.1 +google-cloud-bigquery==3.36.0 pandas-gbq==0.29.2 -pandas==2.3.1 +pandas==2.3.2 pyarrow==21.0.0 From 1a039b151712d8077552c995eb156e5c5e75c3dc Mon Sep 17 00:00:00 2001 From: Lingqing Gan Date: Wed, 17 Sep 2025 16:00:24 -0700 Subject: [PATCH 495/519] test: update prerelease installation path (#948) --- packages/pandas-gbq/noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 46ed30076734..e2e9f7234e97 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -346,7 +346,7 @@ def prerelease(session): # https://github.com/googleapis/python-bigquery-pandas/issues/854 session.install( "https://github.com/googleapis/python-bigquery/archive/main.zip", - "https://github.com/googleapis/python-bigquery-storage/archive/main.zip", + "git+https://github.com/googleapis/google-cloud-python.git@main#subdirectory=packages/google-cloud-bigquery-storage", ) # Because we test minimum dependency versions on the minimum Python From b90eb9f5c81df6f5faedf42623ba0f9fec8a22e4 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Thu, 18 Sep 2025 02:30:43 +0200 Subject: [PATCH 496/519] chore(deps): update all dependencies (#947) Co-authored-by: Anthonios Partheniou --- packages/pandas-gbq/samples/snippets/requirements-test.txt | 2 +- packages/pandas-gbq/samples/snippets/requirements.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/pandas-gbq/samples/snippets/requirements-test.txt b/packages/pandas-gbq/samples/snippets/requirements-test.txt index 2c7482f89a4b..4ad485fb3da5 100644 --- a/packages/pandas-gbq/samples/snippets/requirements-test.txt +++ b/packages/pandas-gbq/samples/snippets/requirements-test.txt @@ -1,2 +1,2 @@ google-cloud-testutils==1.6.4 -pytest==8.4.1 +pytest==8.4.2 diff --git a/packages/pandas-gbq/samples/snippets/requirements.txt b/packages/pandas-gbq/samples/snippets/requirements.txt index 51d8b62ba524..cf4ec5dfc7c3 100644 --- a/packages/pandas-gbq/samples/snippets/requirements.txt +++ b/packages/pandas-gbq/samples/snippets/requirements.txt @@ -1,5 +1,5 @@ -google-cloud-bigquery-storage==2.32.0 -google-cloud-bigquery==3.36.0 +google-cloud-bigquery-storage==2.33.1 +google-cloud-bigquery==3.38.0 pandas-gbq==0.29.2 pandas==2.3.2 pyarrow==21.0.0 From 446ff8a1ba8414e6152422a27c6311c58674fdf1 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 18 Sep 2025 01:00:07 +0000 Subject: [PATCH 497/519] Trigger owlbot post-processor --- owl-bot-staging/pandas-gbq/pandas-gbq/pandas-gbq.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 owl-bot-staging/pandas-gbq/pandas-gbq/pandas-gbq.txt diff --git a/owl-bot-staging/pandas-gbq/pandas-gbq/pandas-gbq.txt b/owl-bot-staging/pandas-gbq/pandas-gbq/pandas-gbq.txt new file mode 100644 index 000000000000..e69de29bb2d1 From 1f9390cd90c267fcf2a3b160f3590e15b650697c Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 18 Sep 2025 01:00:15 +0000 Subject: [PATCH 498/519] build: pandas-gbq migration: adjust owlbot-related files --- .../pandas-gbq/{.github => }/.OwlBot.yaml | 2 - packages/pandas-gbq/.github/.OwlBot.lock.yaml | 17 - packages/pandas-gbq/.github/CODEOWNERS | 12 - packages/pandas-gbq/.github/CONTRIBUTING.md | 28 - .../.github/ISSUE_TEMPLATE/bug_report.md | 43 -- .../.github/ISSUE_TEMPLATE/feature_request.md | 18 - .../.github/ISSUE_TEMPLATE/support_request.md | 7 - .../.github/PULL_REQUEST_TEMPLATE.md | 7 - packages/pandas-gbq/.github/auto-approve.yml | 3 - packages/pandas-gbq/.github/auto-label.yaml | 20 - packages/pandas-gbq/.github/blunderbuss.yml | 20 - .../.github/header-checker-lint.yml | 15 - .../pandas-gbq/.github/release-please.yml | 2 - .../pandas-gbq/.github/release-trigger.yml | 2 - packages/pandas-gbq/.github/snippet-bot.yml | 0 .../.github/sync-repo-settings.yaml | 39 -- .../pandas-gbq/.github/workflows/docs.yml | 38 -- .../pandas-gbq/.github/workflows/lint.yml | 25 - .../pandas-gbq/.github/workflows/unittest.yml | 61 --- packages/pandas-gbq/.kokoro/build.sh | 60 --- .../pandas-gbq/.kokoro/continuous/common.cfg | 27 - .../.kokoro/continuous/continuous.cfg | 1 - .../.kokoro/continuous/prerelease-deps.cfg | 7 - .../.kokoro/continuous/prerelease.cfg | 7 - .../pandas-gbq/.kokoro/populate-secrets.sh | 43 -- .../pandas-gbq/.kokoro/presubmit/common.cfg | 27 - .../.kokoro/presubmit/prerelease-deps.cfg | 7 - .../.kokoro/presubmit/prerelease.cfg | 7 - .../.kokoro/presubmit/presubmit.cfg | 7 - .../.kokoro/presubmit/system-3.13.cfg | 7 - .../.kokoro/presubmit/system-3.9.cfg | 7 - .../.kokoro/samples/lint/common.cfg | 34 -- .../.kokoro/samples/lint/continuous.cfg | 6 - .../.kokoro/samples/lint/periodic.cfg | 6 - .../.kokoro/samples/lint/presubmit.cfg | 6 - .../.kokoro/samples/python3.10/common.cfg | 40 -- .../.kokoro/samples/python3.10/continuous.cfg | 6 - .../samples/python3.10/periodic-head.cfg | 11 - .../.kokoro/samples/python3.10/periodic.cfg | 6 - .../.kokoro/samples/python3.10/presubmit.cfg | 6 - .../.kokoro/samples/python3.11/common.cfg | 40 -- .../.kokoro/samples/python3.11/continuous.cfg | 6 - .../samples/python3.11/periodic-head.cfg | 11 - .../.kokoro/samples/python3.11/periodic.cfg | 6 - .../.kokoro/samples/python3.11/presubmit.cfg | 6 - .../.kokoro/samples/python3.12/common.cfg | 40 -- .../.kokoro/samples/python3.12/continuous.cfg | 6 - .../samples/python3.12/periodic-head.cfg | 11 - .../.kokoro/samples/python3.12/periodic.cfg | 6 - .../.kokoro/samples/python3.12/presubmit.cfg | 6 - .../.kokoro/samples/python3.13/common.cfg | 40 -- .../.kokoro/samples/python3.13/continuous.cfg | 6 - .../samples/python3.13/periodic-head.cfg | 11 - .../.kokoro/samples/python3.13/periodic.cfg | 6 - .../.kokoro/samples/python3.13/presubmit.cfg | 6 - .../.kokoro/samples/python3.7/common.cfg | 40 -- .../.kokoro/samples/python3.7/continuous.cfg | 6 - .../samples/python3.7/periodic-head.cfg | 11 - .../.kokoro/samples/python3.7/periodic.cfg | 6 - .../.kokoro/samples/python3.7/presubmit.cfg | 6 - .../.kokoro/samples/python3.8/common.cfg | 40 -- .../.kokoro/samples/python3.8/continuous.cfg | 6 - .../samples/python3.8/periodic-head.cfg | 11 - .../.kokoro/samples/python3.8/periodic.cfg | 6 - .../.kokoro/samples/python3.8/presubmit.cfg | 6 - .../.kokoro/samples/python3.9/common.cfg | 40 -- .../.kokoro/samples/python3.9/continuous.cfg | 6 - .../samples/python3.9/periodic-head.cfg | 11 - .../.kokoro/samples/python3.9/periodic.cfg | 6 - .../.kokoro/samples/python3.9/presubmit.cfg | 6 - .../.kokoro/test-samples-against-head.sh | 26 - .../pandas-gbq/.kokoro/test-samples-impl.sh | 103 ---- packages/pandas-gbq/.kokoro/test-samples.sh | 44 -- packages/pandas-gbq/.kokoro/trampoline.sh | 28 - packages/pandas-gbq/.kokoro/trampoline_v2.sh | 487 ------------------ packages/pandas-gbq/.trampolinerc | 61 --- packages/pandas-gbq/docs/changelog.md | 1 - packages/pandas-gbq/owlbot.py | 95 ---- 78 files changed, 1946 deletions(-) rename packages/pandas-gbq/{.github => }/.OwlBot.yaml (89%) delete mode 100644 packages/pandas-gbq/.github/.OwlBot.lock.yaml delete mode 100644 packages/pandas-gbq/.github/CODEOWNERS delete mode 100644 packages/pandas-gbq/.github/CONTRIBUTING.md delete mode 100644 packages/pandas-gbq/.github/ISSUE_TEMPLATE/bug_report.md delete mode 100644 packages/pandas-gbq/.github/ISSUE_TEMPLATE/feature_request.md delete mode 100644 packages/pandas-gbq/.github/ISSUE_TEMPLATE/support_request.md delete mode 100644 packages/pandas-gbq/.github/PULL_REQUEST_TEMPLATE.md delete mode 100644 packages/pandas-gbq/.github/auto-approve.yml delete mode 100644 packages/pandas-gbq/.github/auto-label.yaml delete mode 100644 packages/pandas-gbq/.github/blunderbuss.yml delete mode 100644 packages/pandas-gbq/.github/header-checker-lint.yml delete mode 100644 packages/pandas-gbq/.github/release-please.yml delete mode 100644 packages/pandas-gbq/.github/release-trigger.yml delete mode 100644 packages/pandas-gbq/.github/snippet-bot.yml delete mode 100644 packages/pandas-gbq/.github/sync-repo-settings.yaml delete mode 100644 packages/pandas-gbq/.github/workflows/docs.yml delete mode 100644 packages/pandas-gbq/.github/workflows/lint.yml delete mode 100644 packages/pandas-gbq/.github/workflows/unittest.yml delete mode 100755 packages/pandas-gbq/.kokoro/build.sh delete mode 100644 packages/pandas-gbq/.kokoro/continuous/common.cfg delete mode 100644 packages/pandas-gbq/.kokoro/continuous/continuous.cfg delete mode 100644 packages/pandas-gbq/.kokoro/continuous/prerelease-deps.cfg delete mode 100644 packages/pandas-gbq/.kokoro/continuous/prerelease.cfg delete mode 100755 packages/pandas-gbq/.kokoro/populate-secrets.sh delete mode 100644 packages/pandas-gbq/.kokoro/presubmit/common.cfg delete mode 100644 packages/pandas-gbq/.kokoro/presubmit/prerelease-deps.cfg delete mode 100644 packages/pandas-gbq/.kokoro/presubmit/prerelease.cfg delete mode 100644 packages/pandas-gbq/.kokoro/presubmit/presubmit.cfg delete mode 100644 packages/pandas-gbq/.kokoro/presubmit/system-3.13.cfg delete mode 100644 packages/pandas-gbq/.kokoro/presubmit/system-3.9.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/lint/common.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/lint/continuous.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/lint/periodic.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/lint/presubmit.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.10/common.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.10/continuous.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.10/periodic-head.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.10/periodic.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.10/presubmit.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.11/common.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.11/continuous.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.11/periodic-head.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.11/periodic.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.11/presubmit.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.12/common.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.12/continuous.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.12/periodic-head.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.12/periodic.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.12/presubmit.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.13/common.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.13/continuous.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.13/periodic-head.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.13/periodic.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.13/presubmit.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.7/common.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.7/continuous.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.7/periodic-head.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.7/periodic.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.7/presubmit.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.8/common.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.8/continuous.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.8/periodic-head.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.8/periodic.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.8/presubmit.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.9/common.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.9/continuous.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.9/periodic-head.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.9/periodic.cfg delete mode 100644 packages/pandas-gbq/.kokoro/samples/python3.9/presubmit.cfg delete mode 100755 packages/pandas-gbq/.kokoro/test-samples-against-head.sh delete mode 100755 packages/pandas-gbq/.kokoro/test-samples-impl.sh delete mode 100755 packages/pandas-gbq/.kokoro/test-samples.sh delete mode 100755 packages/pandas-gbq/.kokoro/trampoline.sh delete mode 100755 packages/pandas-gbq/.kokoro/trampoline_v2.sh delete mode 100644 packages/pandas-gbq/.trampolinerc delete mode 120000 packages/pandas-gbq/docs/changelog.md delete mode 100644 packages/pandas-gbq/owlbot.py diff --git a/packages/pandas-gbq/.github/.OwlBot.yaml b/packages/pandas-gbq/.OwlBot.yaml similarity index 89% rename from packages/pandas-gbq/.github/.OwlBot.yaml rename to packages/pandas-gbq/.OwlBot.yaml index 33779d65e446..68b0f07b1488 100644 --- a/packages/pandas-gbq/.github/.OwlBot.yaml +++ b/packages/pandas-gbq/.OwlBot.yaml @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -docker: - image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest begin-after-commit-hash: 1afeb53252641dc35a421fa5acc59e2f3229ad6d diff --git a/packages/pandas-gbq/.github/.OwlBot.lock.yaml b/packages/pandas-gbq/.github/.OwlBot.lock.yaml deleted file mode 100644 index c631e1f7d7e9..000000000000 --- a/packages/pandas-gbq/.github/.OwlBot.lock.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -docker: - image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:5581906b957284864632cde4e9c51d1cc66b0094990b27e689132fe5cd036046 -# created: 2025-03-05 diff --git a/packages/pandas-gbq/.github/CODEOWNERS b/packages/pandas-gbq/.github/CODEOWNERS deleted file mode 100644 index 24c0ca968a01..000000000000 --- a/packages/pandas-gbq/.github/CODEOWNERS +++ /dev/null @@ -1,12 +0,0 @@ -# Code owners file. -# This file controls who is tagged for review for any given pull request. -# -# For syntax help see: -# https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners#codeowners-syntax -# Note: This file is autogenerated. To make changes to the codeowner team, please update .repo-metadata.json. - -# @googleapis/yoshi-python @googleapis/api-bigquery @googleapis/api-bigquery-dataframe are the default owners for changes in this repo -* @googleapis/yoshi-python @googleapis/api-bigquery @googleapis/api-bigquery-dataframe - -# @googleapis/python-samples-reviewers @googleapis/api-bigquery @googleapis/api-bigquery-dataframe are the default owners for samples changes -/samples/ @googleapis/python-samples-reviewers @googleapis/api-bigquery @googleapis/api-bigquery-dataframe diff --git a/packages/pandas-gbq/.github/CONTRIBUTING.md b/packages/pandas-gbq/.github/CONTRIBUTING.md deleted file mode 100644 index 939e5341e74d..000000000000 --- a/packages/pandas-gbq/.github/CONTRIBUTING.md +++ /dev/null @@ -1,28 +0,0 @@ -# How to Contribute - -We'd love to accept your patches and contributions to this project. There are -just a few small guidelines you need to follow. - -## Contributor License Agreement - -Contributions to this project must be accompanied by a Contributor License -Agreement. You (or your employer) retain the copyright to your contribution; -this simply gives us permission to use and redistribute your contributions as -part of the project. Head over to to see -your current agreements on file or to sign a new one. - -You generally only need to submit a CLA once, so if you've already submitted one -(even if it was for a different project), you probably don't need to do it -again. - -## Code reviews - -All submissions, including submissions by project members, require review. We -use GitHub pull requests for this purpose. Consult -[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more -information on using pull requests. - -## Community Guidelines - -This project follows [Google's Open Source Community -Guidelines](https://opensource.google.com/conduct/). diff --git a/packages/pandas-gbq/.github/ISSUE_TEMPLATE/bug_report.md b/packages/pandas-gbq/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index 312388ed3446..000000000000 --- a/packages/pandas-gbq/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,43 +0,0 @@ ---- -name: Bug report -about: Create a report to help us improve - ---- - -Thanks for stopping by to let us know something could be better! - -**PLEASE READ**: If you have a support contract with Google, please create an issue in the [support console](https://cloud.google.com/support/) instead of filing on GitHub. This will ensure a timely response. - -Please run down the following list and make sure you've tried the usual "quick fixes": - - - Search the issues already opened: https://github.com/googleapis/python-bigquery-pandas/issues - - Search StackOverflow: https://stackoverflow.com/questions/tagged/google-cloud-platform+python - -If you are still having issues, please be sure to include as much information as possible: - -#### Environment details - - - OS type and version: - - Python version: `python --version` - - pip version: `pip --version` - - `pandas-gbq` version: `pip show pandas-gbq` - -#### Steps to reproduce - - 1. ? - 2. ? - -#### Code example - -```python -# example -``` - -#### Stack trace -``` -# example -``` - -Making sure to follow these steps will guarantee the quickest resolution possible. - -Thanks! diff --git a/packages/pandas-gbq/.github/ISSUE_TEMPLATE/feature_request.md b/packages/pandas-gbq/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index 6365857f33c6..000000000000 --- a/packages/pandas-gbq/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -name: Feature request -about: Suggest an idea for this library - ---- - -Thanks for stopping by to let us know something could be better! - -**PLEASE READ**: If you have a support contract with Google, please create an issue in the [support console](https://cloud.google.com/support/) instead of filing on GitHub. This will ensure a timely response. - - **Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] - **Describe the solution you'd like** -A clear and concise description of what you want to happen. - **Describe alternatives you've considered** -A clear and concise description of any alternative solutions or features you've considered. - **Additional context** -Add any other context or screenshots about the feature request here. diff --git a/packages/pandas-gbq/.github/ISSUE_TEMPLATE/support_request.md b/packages/pandas-gbq/.github/ISSUE_TEMPLATE/support_request.md deleted file mode 100644 index 995869032125..000000000000 --- a/packages/pandas-gbq/.github/ISSUE_TEMPLATE/support_request.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: Support request -about: If you have a support contract with Google, please create an issue in the Google Cloud Support console. - ---- - -**PLEASE READ**: If you have a support contract with Google, please create an issue in the [support console](https://cloud.google.com/support/) instead of filing on GitHub. This will ensure a timely response. diff --git a/packages/pandas-gbq/.github/PULL_REQUEST_TEMPLATE.md b/packages/pandas-gbq/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index 09b516833fa8..000000000000 --- a/packages/pandas-gbq/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,7 +0,0 @@ -Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: -- [ ] Make sure to open an issue as a [bug/issue](https://github.com/googleapis/python-bigquery-pandas/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea -- [ ] Ensure the tests and linter pass -- [ ] Code coverage does not decrease (if any source code was changed) -- [ ] Appropriate docs were updated (if necessary) - -Fixes # 🦕 diff --git a/packages/pandas-gbq/.github/auto-approve.yml b/packages/pandas-gbq/.github/auto-approve.yml deleted file mode 100644 index 311ebbb853a9..000000000000 --- a/packages/pandas-gbq/.github/auto-approve.yml +++ /dev/null @@ -1,3 +0,0 @@ -# https://github.com/googleapis/repo-automation-bots/tree/main/packages/auto-approve -processes: - - "OwlBotTemplateChanges" diff --git a/packages/pandas-gbq/.github/auto-label.yaml b/packages/pandas-gbq/.github/auto-label.yaml deleted file mode 100644 index 21786a4eb085..000000000000 --- a/packages/pandas-gbq/.github/auto-label.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -requestsize: - enabled: true - -path: - pullrequest: true - paths: - samples: "samples" diff --git a/packages/pandas-gbq/.github/blunderbuss.yml b/packages/pandas-gbq/.github/blunderbuss.yml deleted file mode 100644 index 6677a65c995a..000000000000 --- a/packages/pandas-gbq/.github/blunderbuss.yml +++ /dev/null @@ -1,20 +0,0 @@ -# Blunderbuss config -# -# This file controls who is assigned for pull requests and issues. -# Note: This file is autogenerated. To make changes to the assignee -# team, please update `codeowner_team` in `.repo-metadata.json`. -assign_issues: - - googleapis/api-bigquery - - googleapis/api-bigquery-dataframe - -assign_issues_by: - - labels: - - "samples" - to: - - googleapis/python-samples-reviewers - - googleapis/api-bigquery - - googleapis/api-bigquery-dataframe - -assign_prs: - - googleapis/api-bigquery - - googleapis/api-bigquery-dataframe diff --git a/packages/pandas-gbq/.github/header-checker-lint.yml b/packages/pandas-gbq/.github/header-checker-lint.yml deleted file mode 100644 index 62cf51af9046..000000000000 --- a/packages/pandas-gbq/.github/header-checker-lint.yml +++ /dev/null @@ -1,15 +0,0 @@ -{"allowedCopyrightHolders": ["pandas-gbq Authors"], - "allowedLicenses": ["Apache-2.0", "MIT", "BSD-3"], - "ignoreFiles": ["**/requirements.txt", "**/requirements-test.txt", "**/__init__.py", "samples/**/constraints.txt", "samples/**/constraints-test.txt"], - "sourceFileExtensions": [ - "ts", - "js", - "java", - "sh", - "Dockerfile", - "yaml", - "py", - "html", - "txt" - ] -} \ No newline at end of file diff --git a/packages/pandas-gbq/.github/release-please.yml b/packages/pandas-gbq/.github/release-please.yml deleted file mode 100644 index 466597e5b196..000000000000 --- a/packages/pandas-gbq/.github/release-please.yml +++ /dev/null @@ -1,2 +0,0 @@ -releaseType: python -handleGHRelease: true diff --git a/packages/pandas-gbq/.github/release-trigger.yml b/packages/pandas-gbq/.github/release-trigger.yml deleted file mode 100644 index 6601e1508e28..000000000000 --- a/packages/pandas-gbq/.github/release-trigger.yml +++ /dev/null @@ -1,2 +0,0 @@ -enabled: true -multiScmName: python-bigquery-pandas diff --git a/packages/pandas-gbq/.github/snippet-bot.yml b/packages/pandas-gbq/.github/snippet-bot.yml deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/packages/pandas-gbq/.github/sync-repo-settings.yaml b/packages/pandas-gbq/.github/sync-repo-settings.yaml deleted file mode 100644 index 130dd1fef4dc..000000000000 --- a/packages/pandas-gbq/.github/sync-repo-settings.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# https://github.com/googleapis/repo-automation-bots/tree/main/packages/sync-repo-settings -# Rules for main branch protection -branchProtectionRules: -# Identifies the protection rule pattern. Name of the branch to be protected. -# Defaults to `main` -- pattern: main - requiresCodeOwnerReviews: true - requiresStrictStatusChecks: true - requiredStatusCheckContexts: - - 'cla/google' - - 'OwlBot Post Processor' - - 'docs' - - 'lint' - - 'unit (3.9)' - - 'unit (3.10)' - - 'unit (3.11)' - - 'unit (3.12)' - - 'unit (3.13)' - - 'cover' - - 'Kokoro' - - 'Samples - Lint' - - 'Samples - Python 3.9' - - 'Samples - Python 3.10' - - 'Samples - Python 3.11' - - 'Samples - Python 3.12' - - 'Samples - Python 3.13' -permissionRules: - - team: actools-python - permission: admin - - team: actools - permission: admin - - team: api-bigquery - permission: push - - team: yoshi-python - permission: push - - team: python-samples-owners - permission: push - - team: python-samples-reviewers - permission: push diff --git a/packages/pandas-gbq/.github/workflows/docs.yml b/packages/pandas-gbq/.github/workflows/docs.yml deleted file mode 100644 index 2833fe98fff0..000000000000 --- a/packages/pandas-gbq/.github/workflows/docs.yml +++ /dev/null @@ -1,38 +0,0 @@ -on: - pull_request: - branches: - - main -name: docs -jobs: - docs: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - name: Install nox - run: | - python -m pip install --upgrade setuptools pip wheel - python -m pip install nox - - name: Run docs - run: | - nox -s docs - docfx: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - name: Install nox - run: | - python -m pip install --upgrade setuptools pip wheel - python -m pip install nox - - name: Run docfx - run: | - nox -s docfx diff --git a/packages/pandas-gbq/.github/workflows/lint.yml b/packages/pandas-gbq/.github/workflows/lint.yml deleted file mode 100644 index 1051da0bdda4..000000000000 --- a/packages/pandas-gbq/.github/workflows/lint.yml +++ /dev/null @@ -1,25 +0,0 @@ -on: - pull_request: - branches: - - main -name: lint -jobs: - lint: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - name: Install nox - run: | - python -m pip install --upgrade setuptools pip wheel - python -m pip install nox - - name: Run lint - run: | - nox -s lint - - name: Run lint_setup_py - run: | - nox -s lint_setup_py diff --git a/packages/pandas-gbq/.github/workflows/unittest.yml b/packages/pandas-gbq/.github/workflows/unittest.yml deleted file mode 100644 index 7137d0ad257f..000000000000 --- a/packages/pandas-gbq/.github/workflows/unittest.yml +++ /dev/null @@ -1,61 +0,0 @@ -on: - pull_request: - branches: - - main -name: unittest -jobs: - unit: - # TODO(https://github.com/googleapis/gapic-generator-python/issues/2303): use `ubuntu-latest` once this bug is fixed. - # Use ubuntu-22.04 until Python 3.7 is removed from the test matrix - # https://docs.github.com/en/actions/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories - runs-on: ubuntu-22.04 - strategy: - matrix: - python: ['3.9', '3.10', '3.11', '3.12', '3.13'] - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python }} - - name: Install nox - run: | - python -m pip install --upgrade setuptools pip wheel - python -m pip install nox - - name: Run unit tests - env: - COVERAGE_FILE: .coverage-${{ matrix.python }} - run: | - nox -s unit-${{ matrix.python }} - - name: Upload coverage results - uses: actions/upload-artifact@v4 - with: - name: coverage-artifact-${{ matrix.python }} - path: .coverage-${{ matrix.python }} - include-hidden-files: true - - cover: - runs-on: ubuntu-latest - needs: - - unit - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - name: Install coverage - run: | - python -m pip install --upgrade setuptools pip wheel - python -m pip install coverage - - name: Download coverage results - uses: actions/download-artifact@v4 - with: - path: .coverage-results/ - - name: Report coverage results - run: | - find .coverage-results -type f -name '*.zip' -exec unzip {} \; - coverage combine .coverage-results/**/.coverage* - coverage report --show-missing --fail-under=96 diff --git a/packages/pandas-gbq/.kokoro/build.sh b/packages/pandas-gbq/.kokoro/build.sh deleted file mode 100755 index d41b45aa1dd0..000000000000 --- a/packages/pandas-gbq/.kokoro/build.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eo pipefail - -CURRENT_DIR=$(dirname "${BASH_SOURCE[0]}") - -if [[ -z "${PROJECT_ROOT:-}" ]]; then - PROJECT_ROOT=$(realpath "${CURRENT_DIR}/..") -fi - -pushd "${PROJECT_ROOT}" - -# Disable buffering, so that the logs stream through. -export PYTHONUNBUFFERED=1 - -# Debug: show build environment -env | grep KOKORO - -# Setup service account credentials. -if [[ -f "${KOKORO_GFILE_DIR}/service-account.json" ]] -then - export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json -fi - -# Setup project id. -if [[ -f "${KOKORO_GFILE_DIR}/project-id.json" ]] -then - export PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json") -fi - -# If this is a continuous build, send the test log to the FlakyBot. -# See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot. -if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"continuous"* ]]; then - cleanup() { - chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot - $KOKORO_GFILE_DIR/linux_amd64/flakybot - } - trap cleanup EXIT HUP -fi - -# If NOX_SESSION is set, it only runs the specified session, -# otherwise run all the sessions. -if [[ -n "${NOX_SESSION:-}" ]]; then - python3 -m nox -s ${NOX_SESSION:-} -else - python3 -m nox -fi diff --git a/packages/pandas-gbq/.kokoro/continuous/common.cfg b/packages/pandas-gbq/.kokoro/continuous/common.cfg deleted file mode 100644 index 4a34509712a2..000000000000 --- a/packages/pandas-gbq/.kokoro/continuous/common.cfg +++ /dev/null @@ -1,27 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Build logs will be here -action { - define_artifacts { - regex: "**/*sponge_log.xml" - } -} - -# Download trampoline resources. -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" - -# Download resources for system tests (service account key, etc.) -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-python" - -# Use the trampoline script to run in docker. -build_file: "python-bigquery-pandas/.kokoro/trampoline.sh" - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/python-multi" -} -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-bigquery-pandas/.kokoro/build.sh" -} diff --git a/packages/pandas-gbq/.kokoro/continuous/continuous.cfg b/packages/pandas-gbq/.kokoro/continuous/continuous.cfg deleted file mode 100644 index 8f43917d92fe..000000000000 --- a/packages/pandas-gbq/.kokoro/continuous/continuous.cfg +++ /dev/null @@ -1 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/continuous/prerelease-deps.cfg b/packages/pandas-gbq/.kokoro/continuous/prerelease-deps.cfg deleted file mode 100644 index 3595fb43f5c0..000000000000 --- a/packages/pandas-gbq/.kokoro/continuous/prerelease-deps.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Only run this nox session. -env_vars: { - key: "NOX_SESSION" - value: "prerelease_deps" -} diff --git a/packages/pandas-gbq/.kokoro/continuous/prerelease.cfg b/packages/pandas-gbq/.kokoro/continuous/prerelease.cfg deleted file mode 100644 index 00bc8678bc4a..000000000000 --- a/packages/pandas-gbq/.kokoro/continuous/prerelease.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Only run this nox session. -env_vars: { - key: "NOX_SESSION" - value: "prerelease" -} diff --git a/packages/pandas-gbq/.kokoro/populate-secrets.sh b/packages/pandas-gbq/.kokoro/populate-secrets.sh deleted file mode 100755 index c435402f473e..000000000000 --- a/packages/pandas-gbq/.kokoro/populate-secrets.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eo pipefail - -function now { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n' ;} -function msg { println "$*" >&2 ;} -function println { printf '%s\n' "$(now) $*" ;} - - -# Populates requested secrets set in SECRET_MANAGER_KEYS from service account: -# kokoro-trampoline@cloud-devrel-kokoro-resources.iam.gserviceaccount.com -SECRET_LOCATION="${KOKORO_GFILE_DIR}/secret_manager" -msg "Creating folder on disk for secrets: ${SECRET_LOCATION}" -mkdir -p ${SECRET_LOCATION} -for key in $(echo ${SECRET_MANAGER_KEYS} | sed "s/,/ /g") -do - msg "Retrieving secret ${key}" - docker run --entrypoint=gcloud \ - --volume=${KOKORO_GFILE_DIR}:${KOKORO_GFILE_DIR} \ - gcr.io/google.com/cloudsdktool/cloud-sdk \ - secrets versions access latest \ - --project cloud-devrel-kokoro-resources \ - --secret ${key} > \ - "${SECRET_LOCATION}/${key}" - if [[ $? == 0 ]]; then - msg "Secret written to ${SECRET_LOCATION}/${key}" - else - msg "Error retrieving secret ${key}" - fi -done diff --git a/packages/pandas-gbq/.kokoro/presubmit/common.cfg b/packages/pandas-gbq/.kokoro/presubmit/common.cfg deleted file mode 100644 index 4a34509712a2..000000000000 --- a/packages/pandas-gbq/.kokoro/presubmit/common.cfg +++ /dev/null @@ -1,27 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Build logs will be here -action { - define_artifacts { - regex: "**/*sponge_log.xml" - } -} - -# Download trampoline resources. -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" - -# Download resources for system tests (service account key, etc.) -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-python" - -# Use the trampoline script to run in docker. -build_file: "python-bigquery-pandas/.kokoro/trampoline.sh" - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/python-multi" -} -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-bigquery-pandas/.kokoro/build.sh" -} diff --git a/packages/pandas-gbq/.kokoro/presubmit/prerelease-deps.cfg b/packages/pandas-gbq/.kokoro/presubmit/prerelease-deps.cfg deleted file mode 100644 index 3595fb43f5c0..000000000000 --- a/packages/pandas-gbq/.kokoro/presubmit/prerelease-deps.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Only run this nox session. -env_vars: { - key: "NOX_SESSION" - value: "prerelease_deps" -} diff --git a/packages/pandas-gbq/.kokoro/presubmit/prerelease.cfg b/packages/pandas-gbq/.kokoro/presubmit/prerelease.cfg deleted file mode 100644 index 00bc8678bc4a..000000000000 --- a/packages/pandas-gbq/.kokoro/presubmit/prerelease.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Only run this nox session. -env_vars: { - key: "NOX_SESSION" - value: "prerelease" -} diff --git a/packages/pandas-gbq/.kokoro/presubmit/presubmit.cfg b/packages/pandas-gbq/.kokoro/presubmit/presubmit.cfg deleted file mode 100644 index f4651182a0de..000000000000 --- a/packages/pandas-gbq/.kokoro/presubmit/presubmit.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Only run the following session(s) -env_vars: { - key: "NOX_SESSION" - value: "blacken lint lint_setup_py docs" -} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/presubmit/system-3.13.cfg b/packages/pandas-gbq/.kokoro/presubmit/system-3.13.cfg deleted file mode 100644 index 3ec53cf9a70c..000000000000 --- a/packages/pandas-gbq/.kokoro/presubmit/system-3.13.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Only run the following session(s) -env_vars: { - key: "NOX_SESSION" - value: "system-3.13" -} diff --git a/packages/pandas-gbq/.kokoro/presubmit/system-3.9.cfg b/packages/pandas-gbq/.kokoro/presubmit/system-3.9.cfg deleted file mode 100644 index be5a81249af7..000000000000 --- a/packages/pandas-gbq/.kokoro/presubmit/system-3.9.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Only run the following session(s) -env_vars: { - key: "NOX_SESSION" - value: "system-3.9" -} diff --git a/packages/pandas-gbq/.kokoro/samples/lint/common.cfg b/packages/pandas-gbq/.kokoro/samples/lint/common.cfg deleted file mode 100644 index 80ead70e5521..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/lint/common.cfg +++ /dev/null @@ -1,34 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Build logs will be here -action { - define_artifacts { - regex: "**/*sponge_log.xml" - } -} - -# Specify which tests to run -env_vars: { - key: "RUN_TESTS_SESSION" - value: "lint" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-bigquery-pandas/.kokoro/test-samples.sh" -} - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" -} - -# Download secrets for samples -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" - -# Download trampoline resources. -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" - -# Use the trampoline script to run in docker. -build_file: "python-bigquery-pandas/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/lint/continuous.cfg b/packages/pandas-gbq/.kokoro/samples/lint/continuous.cfg deleted file mode 100644 index a1c8d9759c88..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/lint/continuous.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/lint/periodic.cfg b/packages/pandas-gbq/.kokoro/samples/lint/periodic.cfg deleted file mode 100644 index 50fec9649732..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/lint/periodic.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "False" -} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/lint/presubmit.cfg b/packages/pandas-gbq/.kokoro/samples/lint/presubmit.cfg deleted file mode 100644 index a1c8d9759c88..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/lint/presubmit.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.10/common.cfg b/packages/pandas-gbq/.kokoro/samples/python3.10/common.cfg deleted file mode 100644 index 3c9e744f27e2..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.10/common.cfg +++ /dev/null @@ -1,40 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Build logs will be here -action { - define_artifacts { - regex: "**/*sponge_log.xml" - } -} - -# Specify which tests to run -env_vars: { - key: "RUN_TESTS_SESSION" - value: "py-3.10" -} - -# Declare build specific Cloud project. -env_vars: { - key: "BUILD_SPECIFIC_GCLOUD_PROJECT" - value: "python-docs-samples-tests-310" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-bigquery-pandas/.kokoro/test-samples.sh" -} - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" -} - -# Download secrets for samples -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" - -# Download trampoline resources. -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" - -# Use the trampoline script to run in docker. -build_file: "python-bigquery-pandas/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.10/continuous.cfg b/packages/pandas-gbq/.kokoro/samples/python3.10/continuous.cfg deleted file mode 100644 index a1c8d9759c88..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.10/continuous.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.10/periodic-head.cfg b/packages/pandas-gbq/.kokoro/samples/python3.10/periodic-head.cfg deleted file mode 100644 index 98efde4dc99d..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.10/periodic-head.cfg +++ /dev/null @@ -1,11 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-bigquery-pandas/.kokoro/test-samples-against-head.sh" -} diff --git a/packages/pandas-gbq/.kokoro/samples/python3.10/periodic.cfg b/packages/pandas-gbq/.kokoro/samples/python3.10/periodic.cfg deleted file mode 100644 index 71cd1e597e38..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.10/periodic.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "False" -} diff --git a/packages/pandas-gbq/.kokoro/samples/python3.10/presubmit.cfg b/packages/pandas-gbq/.kokoro/samples/python3.10/presubmit.cfg deleted file mode 100644 index a1c8d9759c88..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.10/presubmit.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.11/common.cfg b/packages/pandas-gbq/.kokoro/samples/python3.11/common.cfg deleted file mode 100644 index b9a38d5d8e26..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.11/common.cfg +++ /dev/null @@ -1,40 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Build logs will be here -action { - define_artifacts { - regex: "**/*sponge_log.xml" - } -} - -# Specify which tests to run -env_vars: { - key: "RUN_TESTS_SESSION" - value: "py-3.11" -} - -# Declare build specific Cloud project. -env_vars: { - key: "BUILD_SPECIFIC_GCLOUD_PROJECT" - value: "python-docs-samples-tests-311" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-bigquery-pandas/.kokoro/test-samples.sh" -} - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" -} - -# Download secrets for samples -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" - -# Download trampoline resources. -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" - -# Use the trampoline script to run in docker. -build_file: "python-bigquery-pandas/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.11/continuous.cfg b/packages/pandas-gbq/.kokoro/samples/python3.11/continuous.cfg deleted file mode 100644 index a1c8d9759c88..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.11/continuous.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.11/periodic-head.cfg b/packages/pandas-gbq/.kokoro/samples/python3.11/periodic-head.cfg deleted file mode 100644 index 98efde4dc99d..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.11/periodic-head.cfg +++ /dev/null @@ -1,11 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-bigquery-pandas/.kokoro/test-samples-against-head.sh" -} diff --git a/packages/pandas-gbq/.kokoro/samples/python3.11/periodic.cfg b/packages/pandas-gbq/.kokoro/samples/python3.11/periodic.cfg deleted file mode 100644 index 71cd1e597e38..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.11/periodic.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "False" -} diff --git a/packages/pandas-gbq/.kokoro/samples/python3.11/presubmit.cfg b/packages/pandas-gbq/.kokoro/samples/python3.11/presubmit.cfg deleted file mode 100644 index a1c8d9759c88..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.11/presubmit.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.12/common.cfg b/packages/pandas-gbq/.kokoro/samples/python3.12/common.cfg deleted file mode 100644 index 17c144c4e601..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.12/common.cfg +++ /dev/null @@ -1,40 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Build logs will be here -action { - define_artifacts { - regex: "**/*sponge_log.xml" - } -} - -# Specify which tests to run -env_vars: { - key: "RUN_TESTS_SESSION" - value: "py-3.12" -} - -# Declare build specific Cloud project. -env_vars: { - key: "BUILD_SPECIFIC_GCLOUD_PROJECT" - value: "python-docs-samples-tests-312" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-bigquery-pandas/.kokoro/test-samples.sh" -} - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" -} - -# Download secrets for samples -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" - -# Download trampoline resources. -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" - -# Use the trampoline script to run in docker. -build_file: "python-bigquery-pandas/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.12/continuous.cfg b/packages/pandas-gbq/.kokoro/samples/python3.12/continuous.cfg deleted file mode 100644 index a1c8d9759c88..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.12/continuous.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.12/periodic-head.cfg b/packages/pandas-gbq/.kokoro/samples/python3.12/periodic-head.cfg deleted file mode 100644 index 98efde4dc99d..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.12/periodic-head.cfg +++ /dev/null @@ -1,11 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-bigquery-pandas/.kokoro/test-samples-against-head.sh" -} diff --git a/packages/pandas-gbq/.kokoro/samples/python3.12/periodic.cfg b/packages/pandas-gbq/.kokoro/samples/python3.12/periodic.cfg deleted file mode 100644 index 71cd1e597e38..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.12/periodic.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "False" -} diff --git a/packages/pandas-gbq/.kokoro/samples/python3.12/presubmit.cfg b/packages/pandas-gbq/.kokoro/samples/python3.12/presubmit.cfg deleted file mode 100644 index a1c8d9759c88..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.12/presubmit.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.13/common.cfg b/packages/pandas-gbq/.kokoro/samples/python3.13/common.cfg deleted file mode 100644 index 37ccc958cbc0..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.13/common.cfg +++ /dev/null @@ -1,40 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Build logs will be here -action { - define_artifacts { - regex: "**/*sponge_log.xml" - } -} - -# Specify which tests to run -env_vars: { - key: "RUN_TESTS_SESSION" - value: "py-3.13" -} - -# Declare build specific Cloud project. -env_vars: { - key: "BUILD_SPECIFIC_GCLOUD_PROJECT" - value: "python-docs-samples-tests-313" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-bigquery-pandas/.kokoro/test-samples.sh" -} - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" -} - -# Download secrets for samples -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" - -# Download trampoline resources. -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" - -# Use the trampoline script to run in docker. -build_file: "python-bigquery-pandas/.kokoro/trampoline_v2.sh" diff --git a/packages/pandas-gbq/.kokoro/samples/python3.13/continuous.cfg b/packages/pandas-gbq/.kokoro/samples/python3.13/continuous.cfg deleted file mode 100644 index a1c8d9759c88..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.13/continuous.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.13/periodic-head.cfg b/packages/pandas-gbq/.kokoro/samples/python3.13/periodic-head.cfg deleted file mode 100644 index 98efde4dc99d..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.13/periodic-head.cfg +++ /dev/null @@ -1,11 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-bigquery-pandas/.kokoro/test-samples-against-head.sh" -} diff --git a/packages/pandas-gbq/.kokoro/samples/python3.13/periodic.cfg b/packages/pandas-gbq/.kokoro/samples/python3.13/periodic.cfg deleted file mode 100644 index 71cd1e597e38..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.13/periodic.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "False" -} diff --git a/packages/pandas-gbq/.kokoro/samples/python3.13/presubmit.cfg b/packages/pandas-gbq/.kokoro/samples/python3.13/presubmit.cfg deleted file mode 100644 index a1c8d9759c88..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.13/presubmit.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.7/common.cfg b/packages/pandas-gbq/.kokoro/samples/python3.7/common.cfg deleted file mode 100644 index be202f330704..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.7/common.cfg +++ /dev/null @@ -1,40 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Build logs will be here -action { - define_artifacts { - regex: "**/*sponge_log.xml" - } -} - -# Specify which tests to run -env_vars: { - key: "RUN_TESTS_SESSION" - value: "py-3.7" -} - -# Declare build specific Cloud project. -env_vars: { - key: "BUILD_SPECIFIC_GCLOUD_PROJECT" - value: "python-docs-samples-tests-py37" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-bigquery-pandas/.kokoro/test-samples.sh" -} - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" -} - -# Download secrets for samples -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" - -# Download trampoline resources. -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" - -# Use the trampoline script to run in docker. -build_file: "python-bigquery-pandas/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.7/continuous.cfg b/packages/pandas-gbq/.kokoro/samples/python3.7/continuous.cfg deleted file mode 100644 index a1c8d9759c88..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.7/continuous.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.7/periodic-head.cfg b/packages/pandas-gbq/.kokoro/samples/python3.7/periodic-head.cfg deleted file mode 100644 index 98efde4dc99d..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.7/periodic-head.cfg +++ /dev/null @@ -1,11 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-bigquery-pandas/.kokoro/test-samples-against-head.sh" -} diff --git a/packages/pandas-gbq/.kokoro/samples/python3.7/periodic.cfg b/packages/pandas-gbq/.kokoro/samples/python3.7/periodic.cfg deleted file mode 100644 index 71cd1e597e38..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.7/periodic.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "False" -} diff --git a/packages/pandas-gbq/.kokoro/samples/python3.7/presubmit.cfg b/packages/pandas-gbq/.kokoro/samples/python3.7/presubmit.cfg deleted file mode 100644 index a1c8d9759c88..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.7/presubmit.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.8/common.cfg b/packages/pandas-gbq/.kokoro/samples/python3.8/common.cfg deleted file mode 100644 index 7424a3b9d52b..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.8/common.cfg +++ /dev/null @@ -1,40 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Build logs will be here -action { - define_artifacts { - regex: "**/*sponge_log.xml" - } -} - -# Specify which tests to run -env_vars: { - key: "RUN_TESTS_SESSION" - value: "py-3.8" -} - -# Declare build specific Cloud project. -env_vars: { - key: "BUILD_SPECIFIC_GCLOUD_PROJECT" - value: "python-docs-samples-tests-py38" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-bigquery-pandas/.kokoro/test-samples.sh" -} - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" -} - -# Download secrets for samples -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" - -# Download trampoline resources. -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" - -# Use the trampoline script to run in docker. -build_file: "python-bigquery-pandas/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.8/continuous.cfg b/packages/pandas-gbq/.kokoro/samples/python3.8/continuous.cfg deleted file mode 100644 index a1c8d9759c88..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.8/continuous.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.8/periodic-head.cfg b/packages/pandas-gbq/.kokoro/samples/python3.8/periodic-head.cfg deleted file mode 100644 index 98efde4dc99d..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.8/periodic-head.cfg +++ /dev/null @@ -1,11 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-bigquery-pandas/.kokoro/test-samples-against-head.sh" -} diff --git a/packages/pandas-gbq/.kokoro/samples/python3.8/periodic.cfg b/packages/pandas-gbq/.kokoro/samples/python3.8/periodic.cfg deleted file mode 100644 index 71cd1e597e38..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.8/periodic.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "False" -} diff --git a/packages/pandas-gbq/.kokoro/samples/python3.8/presubmit.cfg b/packages/pandas-gbq/.kokoro/samples/python3.8/presubmit.cfg deleted file mode 100644 index a1c8d9759c88..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.8/presubmit.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.9/common.cfg b/packages/pandas-gbq/.kokoro/samples/python3.9/common.cfg deleted file mode 100644 index c0948d2112b0..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.9/common.cfg +++ /dev/null @@ -1,40 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Build logs will be here -action { - define_artifacts { - regex: "**/*sponge_log.xml" - } -} - -# Specify which tests to run -env_vars: { - key: "RUN_TESTS_SESSION" - value: "py-3.9" -} - -# Declare build specific Cloud project. -env_vars: { - key: "BUILD_SPECIFIC_GCLOUD_PROJECT" - value: "python-docs-samples-tests-py39" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-bigquery-pandas/.kokoro/test-samples.sh" -} - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" -} - -# Download secrets for samples -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" - -# Download trampoline resources. -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" - -# Use the trampoline script to run in docker. -build_file: "python-bigquery-pandas/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.9/continuous.cfg b/packages/pandas-gbq/.kokoro/samples/python3.9/continuous.cfg deleted file mode 100644 index a1c8d9759c88..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.9/continuous.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/samples/python3.9/periodic-head.cfg b/packages/pandas-gbq/.kokoro/samples/python3.9/periodic-head.cfg deleted file mode 100644 index 98efde4dc99d..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.9/periodic-head.cfg +++ /dev/null @@ -1,11 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-bigquery-pandas/.kokoro/test-samples-against-head.sh" -} diff --git a/packages/pandas-gbq/.kokoro/samples/python3.9/periodic.cfg b/packages/pandas-gbq/.kokoro/samples/python3.9/periodic.cfg deleted file mode 100644 index 71cd1e597e38..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.9/periodic.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "False" -} diff --git a/packages/pandas-gbq/.kokoro/samples/python3.9/presubmit.cfg b/packages/pandas-gbq/.kokoro/samples/python3.9/presubmit.cfg deleted file mode 100644 index a1c8d9759c88..000000000000 --- a/packages/pandas-gbq/.kokoro/samples/python3.9/presubmit.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/test-samples-against-head.sh b/packages/pandas-gbq/.kokoro/test-samples-against-head.sh deleted file mode 100755 index e9d8bd79a644..000000000000 --- a/packages/pandas-gbq/.kokoro/test-samples-against-head.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# A customized test runner for samples. -# -# For periodic builds, you can specify this file for testing against head. - -# `-e` enables the script to automatically fail when a command fails -# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero -set -eo pipefail -# Enables `**` to include files nested inside sub-folders -shopt -s globstar - -exec .kokoro/test-samples-impl.sh diff --git a/packages/pandas-gbq/.kokoro/test-samples-impl.sh b/packages/pandas-gbq/.kokoro/test-samples-impl.sh deleted file mode 100755 index 53e365bc4e79..000000000000 --- a/packages/pandas-gbq/.kokoro/test-samples-impl.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# `-e` enables the script to automatically fail when a command fails -# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero -set -eo pipefail -# Enables `**` to include files nested inside sub-folders -shopt -s globstar - -# Exit early if samples don't exist -if ! find samples -name 'requirements.txt' | grep -q .; then - echo "No tests run. './samples/**/requirements.txt' not found" - exit 0 -fi - -# Disable buffering, so that the logs stream through. -export PYTHONUNBUFFERED=1 - -# Debug: show build environment -env | grep KOKORO - -# Install nox -# `virtualenv==20.26.6` is added for Python 3.7 compatibility -python3.9 -m pip install --upgrade --quiet nox virtualenv==20.26.6 - -# Use secrets acessor service account to get secrets -if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then - gcloud auth activate-service-account \ - --key-file="${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" \ - --project="cloud-devrel-kokoro-resources" -fi - -# This script will create 3 files: -# - testing/test-env.sh -# - testing/service-account.json -# - testing/client-secrets.json -./scripts/decrypt-secrets.sh - -source ./testing/test-env.sh -export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/testing/service-account.json - -# For cloud-run session, we activate the service account for gcloud sdk. -gcloud auth activate-service-account \ - --key-file "${GOOGLE_APPLICATION_CREDENTIALS}" - -export GOOGLE_CLIENT_SECRETS=$(pwd)/testing/client-secrets.json - -echo -e "\n******************** TESTING PROJECTS ********************" - -# Switch to 'fail at end' to allow all tests to complete before exiting. -set +e -# Use RTN to return a non-zero value if the test fails. -RTN=0 -ROOT=$(pwd) -# Find all requirements.txt in the samples directory (may break on whitespace). -for file in samples/**/requirements.txt; do - cd "$ROOT" - # Navigate to the project folder. - file=$(dirname "$file") - cd "$file" - - echo "------------------------------------------------------------" - echo "- testing $file" - echo "------------------------------------------------------------" - - # Use nox to execute the tests for the project. - python3.9 -m nox -s "$RUN_TESTS_SESSION" - EXIT=$? - - # If this is a periodic build, send the test log to the FlakyBot. - # See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot. - if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then - chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot - $KOKORO_GFILE_DIR/linux_amd64/flakybot - fi - - if [[ $EXIT -ne 0 ]]; then - RTN=1 - echo -e "\n Testing failed: Nox returned a non-zero exit code. \n" - else - echo -e "\n Testing completed.\n" - fi - -done -cd "$ROOT" - -# Workaround for Kokoro permissions issue: delete secrets -rm testing/{test-env.sh,client-secrets.json,service-account.json} - -exit "$RTN" diff --git a/packages/pandas-gbq/.kokoro/test-samples.sh b/packages/pandas-gbq/.kokoro/test-samples.sh deleted file mode 100755 index 7933d820149a..000000000000 --- a/packages/pandas-gbq/.kokoro/test-samples.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# The default test runner for samples. -# -# For periodic builds, we rewinds the repo to the latest release, and -# run test-samples-impl.sh. - -# `-e` enables the script to automatically fail when a command fails -# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero -set -eo pipefail -# Enables `**` to include files nested inside sub-folders -shopt -s globstar - -# Run periodic samples tests at latest release -if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then - # preserving the test runner implementation. - cp .kokoro/test-samples-impl.sh "${TMPDIR}/test-samples-impl.sh" - echo "--- IMPORTANT IMPORTANT IMPORTANT ---" - echo "Now we rewind the repo back to the latest release..." - LATEST_RELEASE=$(git describe --abbrev=0 --tags) - git checkout $LATEST_RELEASE - echo "The current head is: " - echo $(git rev-parse --verify HEAD) - echo "--- IMPORTANT IMPORTANT IMPORTANT ---" - # move back the test runner implementation if there's no file. - if [ ! -f .kokoro/test-samples-impl.sh ]; then - cp "${TMPDIR}/test-samples-impl.sh" .kokoro/test-samples-impl.sh - fi -fi - -exec .kokoro/test-samples-impl.sh diff --git a/packages/pandas-gbq/.kokoro/trampoline.sh b/packages/pandas-gbq/.kokoro/trampoline.sh deleted file mode 100755 index 48f79699706e..000000000000 --- a/packages/pandas-gbq/.kokoro/trampoline.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eo pipefail - -# Always run the cleanup script, regardless of the success of bouncing into -# the container. -function cleanup() { - chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh - ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh - echo "cleanup"; -} -trap cleanup EXIT - -$(dirname $0)/populate-secrets.sh # Secret Manager secrets. -python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py" \ No newline at end of file diff --git a/packages/pandas-gbq/.kokoro/trampoline_v2.sh b/packages/pandas-gbq/.kokoro/trampoline_v2.sh deleted file mode 100755 index 35fa529231dc..000000000000 --- a/packages/pandas-gbq/.kokoro/trampoline_v2.sh +++ /dev/null @@ -1,487 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# trampoline_v2.sh -# -# This script does 3 things. -# -# 1. Prepare the Docker image for the test -# 2. Run the Docker with appropriate flags to run the test -# 3. Upload the newly built Docker image -# -# in a way that is somewhat compatible with trampoline_v1. -# -# To run this script, first download few files from gcs to /dev/shm. -# (/dev/shm is passed into the container as KOKORO_GFILE_DIR). -# -# gsutil cp gs://cloud-devrel-kokoro-resources/python-docs-samples/secrets_viewer_service_account.json /dev/shm -# gsutil cp gs://cloud-devrel-kokoro-resources/python-docs-samples/automl_secrets.txt /dev/shm -# -# Then run the script. -# .kokoro/trampoline_v2.sh -# -# These environment variables are required: -# TRAMPOLINE_IMAGE: The docker image to use. -# TRAMPOLINE_DOCKERFILE: The location of the Dockerfile. -# -# You can optionally change these environment variables: -# TRAMPOLINE_IMAGE_UPLOAD: -# (true|false): Whether to upload the Docker image after the -# successful builds. -# TRAMPOLINE_BUILD_FILE: The script to run in the docker container. -# TRAMPOLINE_WORKSPACE: The workspace path in the docker container. -# Defaults to /workspace. -# Potentially there are some repo specific envvars in .trampolinerc in -# the project root. - - -set -euo pipefail - -TRAMPOLINE_VERSION="2.0.5" - -if command -v tput >/dev/null && [[ -n "${TERM:-}" ]]; then - readonly IO_COLOR_RED="$(tput setaf 1)" - readonly IO_COLOR_GREEN="$(tput setaf 2)" - readonly IO_COLOR_YELLOW="$(tput setaf 3)" - readonly IO_COLOR_RESET="$(tput sgr0)" -else - readonly IO_COLOR_RED="" - readonly IO_COLOR_GREEN="" - readonly IO_COLOR_YELLOW="" - readonly IO_COLOR_RESET="" -fi - -function function_exists { - [ $(LC_ALL=C type -t $1)"" == "function" ] -} - -# Logs a message using the given color. The first argument must be one -# of the IO_COLOR_* variables defined above, such as -# "${IO_COLOR_YELLOW}". The remaining arguments will be logged in the -# given color. The log message will also have an RFC-3339 timestamp -# prepended (in UTC). You can disable the color output by setting -# TERM=vt100. -function log_impl() { - local color="$1" - shift - local timestamp="$(date -u "+%Y-%m-%dT%H:%M:%SZ")" - echo "================================================================" - echo "${color}${timestamp}:" "$@" "${IO_COLOR_RESET}" - echo "================================================================" -} - -# Logs the given message with normal coloring and a timestamp. -function log() { - log_impl "${IO_COLOR_RESET}" "$@" -} - -# Logs the given message in green with a timestamp. -function log_green() { - log_impl "${IO_COLOR_GREEN}" "$@" -} - -# Logs the given message in yellow with a timestamp. -function log_yellow() { - log_impl "${IO_COLOR_YELLOW}" "$@" -} - -# Logs the given message in red with a timestamp. -function log_red() { - log_impl "${IO_COLOR_RED}" "$@" -} - -readonly tmpdir=$(mktemp -d -t ci-XXXXXXXX) -readonly tmphome="${tmpdir}/h" -mkdir -p "${tmphome}" - -function cleanup() { - rm -rf "${tmpdir}" -} -trap cleanup EXIT - -RUNNING_IN_CI="${RUNNING_IN_CI:-false}" - -# The workspace in the container, defaults to /workspace. -TRAMPOLINE_WORKSPACE="${TRAMPOLINE_WORKSPACE:-/workspace}" - -pass_down_envvars=( - # TRAMPOLINE_V2 variables. - # Tells scripts whether they are running as part of CI or not. - "RUNNING_IN_CI" - # Indicates which CI system we're in. - "TRAMPOLINE_CI" - # Indicates the version of the script. - "TRAMPOLINE_VERSION" -) - -log_yellow "Building with Trampoline ${TRAMPOLINE_VERSION}" - -# Detect which CI systems we're in. If we're in any of the CI systems -# we support, `RUNNING_IN_CI` will be true and `TRAMPOLINE_CI` will be -# the name of the CI system. Both envvars will be passing down to the -# container for telling which CI system we're in. -if [[ -n "${KOKORO_BUILD_ID:-}" ]]; then - # descriptive env var for indicating it's on CI. - RUNNING_IN_CI="true" - TRAMPOLINE_CI="kokoro" - if [[ "${TRAMPOLINE_USE_LEGACY_SERVICE_ACCOUNT:-}" == "true" ]]; then - if [[ ! -f "${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json" ]]; then - log_red "${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json does not exist. Did you forget to mount cloud-devrel-kokoro-resources/trampoline? Aborting." - exit 1 - fi - # This service account will be activated later. - TRAMPOLINE_SERVICE_ACCOUNT="${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json" - else - if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then - gcloud auth list - fi - log_yellow "Configuring Container Registry access" - gcloud auth configure-docker --quiet - fi - pass_down_envvars+=( - # KOKORO dynamic variables. - "KOKORO_BUILD_NUMBER" - "KOKORO_BUILD_ID" - "KOKORO_JOB_NAME" - "KOKORO_GIT_COMMIT" - "KOKORO_GITHUB_COMMIT" - "KOKORO_GITHUB_PULL_REQUEST_NUMBER" - "KOKORO_GITHUB_PULL_REQUEST_COMMIT" - # For FlakyBot - "KOKORO_GITHUB_COMMIT_URL" - "KOKORO_GITHUB_PULL_REQUEST_URL" - ) -elif [[ "${TRAVIS:-}" == "true" ]]; then - RUNNING_IN_CI="true" - TRAMPOLINE_CI="travis" - pass_down_envvars+=( - "TRAVIS_BRANCH" - "TRAVIS_BUILD_ID" - "TRAVIS_BUILD_NUMBER" - "TRAVIS_BUILD_WEB_URL" - "TRAVIS_COMMIT" - "TRAVIS_COMMIT_MESSAGE" - "TRAVIS_COMMIT_RANGE" - "TRAVIS_JOB_NAME" - "TRAVIS_JOB_NUMBER" - "TRAVIS_JOB_WEB_URL" - "TRAVIS_PULL_REQUEST" - "TRAVIS_PULL_REQUEST_BRANCH" - "TRAVIS_PULL_REQUEST_SHA" - "TRAVIS_PULL_REQUEST_SLUG" - "TRAVIS_REPO_SLUG" - "TRAVIS_SECURE_ENV_VARS" - "TRAVIS_TAG" - ) -elif [[ -n "${GITHUB_RUN_ID:-}" ]]; then - RUNNING_IN_CI="true" - TRAMPOLINE_CI="github-workflow" - pass_down_envvars+=( - "GITHUB_WORKFLOW" - "GITHUB_RUN_ID" - "GITHUB_RUN_NUMBER" - "GITHUB_ACTION" - "GITHUB_ACTIONS" - "GITHUB_ACTOR" - "GITHUB_REPOSITORY" - "GITHUB_EVENT_NAME" - "GITHUB_EVENT_PATH" - "GITHUB_SHA" - "GITHUB_REF" - "GITHUB_HEAD_REF" - "GITHUB_BASE_REF" - ) -elif [[ "${CIRCLECI:-}" == "true" ]]; then - RUNNING_IN_CI="true" - TRAMPOLINE_CI="circleci" - pass_down_envvars+=( - "CIRCLE_BRANCH" - "CIRCLE_BUILD_NUM" - "CIRCLE_BUILD_URL" - "CIRCLE_COMPARE_URL" - "CIRCLE_JOB" - "CIRCLE_NODE_INDEX" - "CIRCLE_NODE_TOTAL" - "CIRCLE_PREVIOUS_BUILD_NUM" - "CIRCLE_PROJECT_REPONAME" - "CIRCLE_PROJECT_USERNAME" - "CIRCLE_REPOSITORY_URL" - "CIRCLE_SHA1" - "CIRCLE_STAGE" - "CIRCLE_USERNAME" - "CIRCLE_WORKFLOW_ID" - "CIRCLE_WORKFLOW_JOB_ID" - "CIRCLE_WORKFLOW_UPSTREAM_JOB_IDS" - "CIRCLE_WORKFLOW_WORKSPACE_ID" - ) -fi - -# Configure the service account for pulling the docker image. -function repo_root() { - local dir="$1" - while [[ ! -d "${dir}/.git" ]]; do - dir="$(dirname "$dir")" - done - echo "${dir}" -} - -# Detect the project root. In CI builds, we assume the script is in -# the git tree and traverse from there, otherwise, traverse from `pwd` -# to find `.git` directory. -if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then - PROGRAM_PATH="$(realpath "$0")" - PROGRAM_DIR="$(dirname "${PROGRAM_PATH}")" - PROJECT_ROOT="$(repo_root "${PROGRAM_DIR}")" -else - PROJECT_ROOT="$(repo_root $(pwd))" -fi - -log_yellow "Changing to the project root: ${PROJECT_ROOT}." -cd "${PROJECT_ROOT}" - -# To support relative path for `TRAMPOLINE_SERVICE_ACCOUNT`, we need -# to use this environment variable in `PROJECT_ROOT`. -if [[ -n "${TRAMPOLINE_SERVICE_ACCOUNT:-}" ]]; then - - mkdir -p "${tmpdir}/gcloud" - gcloud_config_dir="${tmpdir}/gcloud" - - log_yellow "Using isolated gcloud config: ${gcloud_config_dir}." - export CLOUDSDK_CONFIG="${gcloud_config_dir}" - - log_yellow "Using ${TRAMPOLINE_SERVICE_ACCOUNT} for authentication." - gcloud auth activate-service-account \ - --key-file "${TRAMPOLINE_SERVICE_ACCOUNT}" - log_yellow "Configuring Container Registry access" - gcloud auth configure-docker --quiet -fi - -required_envvars=( - # The basic trampoline configurations. - "TRAMPOLINE_IMAGE" - "TRAMPOLINE_BUILD_FILE" -) - -if [[ -f "${PROJECT_ROOT}/.trampolinerc" ]]; then - source "${PROJECT_ROOT}/.trampolinerc" -fi - -log_yellow "Checking environment variables." -for e in "${required_envvars[@]}" -do - if [[ -z "${!e:-}" ]]; then - log "Missing ${e} env var. Aborting." - exit 1 - fi -done - -# We want to support legacy style TRAMPOLINE_BUILD_FILE used with V1 -# script: e.g. "github/repo-name/.kokoro/run_tests.sh" -TRAMPOLINE_BUILD_FILE="${TRAMPOLINE_BUILD_FILE#github/*/}" -log_yellow "Using TRAMPOLINE_BUILD_FILE: ${TRAMPOLINE_BUILD_FILE}" - -# ignore error on docker operations and test execution -set +e - -log_yellow "Preparing Docker image." -# We only download the docker image in CI builds. -if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then - # Download the docker image specified by `TRAMPOLINE_IMAGE` - - # We may want to add --max-concurrent-downloads flag. - - log_yellow "Start pulling the Docker image: ${TRAMPOLINE_IMAGE}." - if docker pull "${TRAMPOLINE_IMAGE}"; then - log_green "Finished pulling the Docker image: ${TRAMPOLINE_IMAGE}." - has_image="true" - else - log_red "Failed pulling the Docker image: ${TRAMPOLINE_IMAGE}." - has_image="false" - fi -else - # For local run, check if we have the image. - if docker images "${TRAMPOLINE_IMAGE}:latest" | grep "${TRAMPOLINE_IMAGE}"; then - has_image="true" - else - has_image="false" - fi -fi - - -# The default user for a Docker container has uid 0 (root). To avoid -# creating root-owned files in the build directory we tell docker to -# use the current user ID. -user_uid="$(id -u)" -user_gid="$(id -g)" -user_name="$(id -un)" - -# To allow docker in docker, we add the user to the docker group in -# the host os. -docker_gid=$(cut -d: -f3 < <(getent group docker)) - -update_cache="false" -if [[ "${TRAMPOLINE_DOCKERFILE:-none}" != "none" ]]; then - # Build the Docker image from the source. - context_dir=$(dirname "${TRAMPOLINE_DOCKERFILE}") - docker_build_flags=( - "-f" "${TRAMPOLINE_DOCKERFILE}" - "-t" "${TRAMPOLINE_IMAGE}" - "--build-arg" "UID=${user_uid}" - "--build-arg" "USERNAME=${user_name}" - ) - if [[ "${has_image}" == "true" ]]; then - docker_build_flags+=("--cache-from" "${TRAMPOLINE_IMAGE}") - fi - - log_yellow "Start building the docker image." - if [[ "${TRAMPOLINE_VERBOSE:-false}" == "true" ]]; then - echo "docker build" "${docker_build_flags[@]}" "${context_dir}" - fi - - # ON CI systems, we want to suppress docker build logs, only - # output the logs when it fails. - if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then - if docker build "${docker_build_flags[@]}" "${context_dir}" \ - > "${tmpdir}/docker_build.log" 2>&1; then - if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then - cat "${tmpdir}/docker_build.log" - fi - - log_green "Finished building the docker image." - update_cache="true" - else - log_red "Failed to build the Docker image, aborting." - log_yellow "Dumping the build logs:" - cat "${tmpdir}/docker_build.log" - exit 1 - fi - else - if docker build "${docker_build_flags[@]}" "${context_dir}"; then - log_green "Finished building the docker image." - update_cache="true" - else - log_red "Failed to build the Docker image, aborting." - exit 1 - fi - fi -else - if [[ "${has_image}" != "true" ]]; then - log_red "We do not have ${TRAMPOLINE_IMAGE} locally, aborting." - exit 1 - fi -fi - -# We use an array for the flags so they are easier to document. -docker_flags=( - # Remove the container after it exists. - "--rm" - - # Use the host network. - "--network=host" - - # Run in priviledged mode. We are not using docker for sandboxing or - # isolation, just for packaging our dev tools. - "--privileged" - - # Run the docker script with the user id. Because the docker image gets to - # write in ${PWD} you typically want this to be your user id. - # To allow docker in docker, we need to use docker gid on the host. - "--user" "${user_uid}:${docker_gid}" - - # Pass down the USER. - "--env" "USER=${user_name}" - - # Mount the project directory inside the Docker container. - "--volume" "${PROJECT_ROOT}:${TRAMPOLINE_WORKSPACE}" - "--workdir" "${TRAMPOLINE_WORKSPACE}" - "--env" "PROJECT_ROOT=${TRAMPOLINE_WORKSPACE}" - - # Mount the temporary home directory. - "--volume" "${tmphome}:/h" - "--env" "HOME=/h" - - # Allow docker in docker. - "--volume" "/var/run/docker.sock:/var/run/docker.sock" - - # Mount the /tmp so that docker in docker can mount the files - # there correctly. - "--volume" "/tmp:/tmp" - # Pass down the KOKORO_GFILE_DIR and KOKORO_KEYSTORE_DIR - # TODO(tmatsuo): This part is not portable. - "--env" "TRAMPOLINE_SECRET_DIR=/secrets" - "--volume" "${KOKORO_GFILE_DIR:-/dev/shm}:/secrets/gfile" - "--env" "KOKORO_GFILE_DIR=/secrets/gfile" - "--volume" "${KOKORO_KEYSTORE_DIR:-/dev/shm}:/secrets/keystore" - "--env" "KOKORO_KEYSTORE_DIR=/secrets/keystore" -) - -# Add an option for nicer output if the build gets a tty. -if [[ -t 0 ]]; then - docker_flags+=("-it") -fi - -# Passing down env vars -for e in "${pass_down_envvars[@]}" -do - if [[ -n "${!e:-}" ]]; then - docker_flags+=("--env" "${e}=${!e}") - fi -done - -# If arguments are given, all arguments will become the commands run -# in the container, otherwise run TRAMPOLINE_BUILD_FILE. -if [[ $# -ge 1 ]]; then - log_yellow "Running the given commands '" "${@:1}" "' in the container." - readonly commands=("${@:1}") - if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then - echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}" - fi - docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}" -else - log_yellow "Running the tests in a Docker container." - docker_flags+=("--entrypoint=${TRAMPOLINE_BUILD_FILE}") - if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then - echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" - fi - docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" -fi - - -test_retval=$? - -if [[ ${test_retval} -eq 0 ]]; then - log_green "Build finished with ${test_retval}" -else - log_red "Build finished with ${test_retval}" -fi - -# Only upload it when the test passes. -if [[ "${update_cache}" == "true" ]] && \ - [[ $test_retval == 0 ]] && \ - [[ "${TRAMPOLINE_IMAGE_UPLOAD:-false}" == "true" ]]; then - log_yellow "Uploading the Docker image." - if docker push "${TRAMPOLINE_IMAGE}"; then - log_green "Finished uploading the Docker image." - else - log_red "Failed uploading the Docker image." - fi - # Call trampoline_after_upload_hook if it's defined. - if function_exists trampoline_after_upload_hook; then - trampoline_after_upload_hook - fi - -fi - -exit "${test_retval}" diff --git a/packages/pandas-gbq/.trampolinerc b/packages/pandas-gbq/.trampolinerc deleted file mode 100644 index 0080152373d5..000000000000 --- a/packages/pandas-gbq/.trampolinerc +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Add required env vars here. -required_envvars+=( -) - -# Add env vars which are passed down into the container here. -pass_down_envvars+=( - "NOX_SESSION" - ############### - # Docs builds - ############### - "STAGING_BUCKET" - "V2_STAGING_BUCKET" - ################## - # Samples builds - ################## - "INSTALL_LIBRARY_FROM_SOURCE" - "RUN_TESTS_SESSION" - "BUILD_SPECIFIC_GCLOUD_PROJECT" - # Target directories. - "RUN_TESTS_DIRS" - # The nox session to run. - "RUN_TESTS_SESSION" -) - -# Prevent unintentional override on the default image. -if [[ "${TRAMPOLINE_IMAGE_UPLOAD:-false}" == "true" ]] && \ - [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then - echo "Please set TRAMPOLINE_IMAGE if you want to upload the Docker image." - exit 1 -fi - -# Define the default value if it makes sense. -if [[ -z "${TRAMPOLINE_IMAGE_UPLOAD:-}" ]]; then - TRAMPOLINE_IMAGE_UPLOAD="" -fi - -if [[ -z "${TRAMPOLINE_IMAGE:-}" ]]; then - TRAMPOLINE_IMAGE="" -fi - -if [[ -z "${TRAMPOLINE_DOCKERFILE:-}" ]]; then - TRAMPOLINE_DOCKERFILE="" -fi - -if [[ -z "${TRAMPOLINE_BUILD_FILE:-}" ]]; then - TRAMPOLINE_BUILD_FILE="" -fi diff --git a/packages/pandas-gbq/docs/changelog.md b/packages/pandas-gbq/docs/changelog.md deleted file mode 120000 index 04c99a55caae..000000000000 --- a/packages/pandas-gbq/docs/changelog.md +++ /dev/null @@ -1 +0,0 @@ -../CHANGELOG.md \ No newline at end of file diff --git a/packages/pandas-gbq/owlbot.py b/packages/pandas-gbq/owlbot.py deleted file mode 100644 index cde35a98b10e..000000000000 --- a/packages/pandas-gbq/owlbot.py +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""This script is used to synthesize generated parts of this library.""" - -import pathlib - -import synthtool as s -from synthtool import gcp -from synthtool.languages import python - -REPO_ROOT = pathlib.Path(__file__).parent.absolute() - -common = gcp.CommonTemplates() - -# ---------------------------------------------------------------------------- -# Add templated files -# ---------------------------------------------------------------------------- - -extras_by_python = { - # Use a middle version of Python to test when no extras are installed. - "3.9": [] -} -extras = ["tqdm", "geopandas"] -templated_files = common.py_library( - default_python_version="3.10", - unit_test_python_versions=["3.9", "3.10", "3.11", "3.12", "3.13"], - system_test_python_versions=["3.9", "3.10", "3.11", "3.12", "3.13"], - cov_level=96, - unit_test_external_dependencies=["freezegun"], - unit_test_extras=extras, - unit_test_extras_by_python=extras_by_python, - system_test_extras=extras, - intersphinx_dependencies={ - "pandas": "https://pandas.pydata.org/pandas-docs/stable/", - "pydata-google-auth": "https://pydata-google-auth.readthedocs.io/en/latest/", - }, -) -s.move( - templated_files, - excludes=[ - # pandas-gbq was originally licensed BSD-3-Clause License - "LICENSE", - # Multi-processing note isn't relevant, as pandas_gbq is responsible for - # creating clients, not the end user. - "docs/multiprocessing.rst", - "noxfile.py", - "README.rst", - ".github/workflows/docs.yml", # to avoid overwriting python version - ".github/workflows/lint.yml", # to avoid overwriting python version - ".github/sync-repo-settings.yaml", - # exclude this file as we have an alternate prerelease.cfg - ".kokoro/presubmit/prerelease-deps.cfg", - ".kokoro/presubmit/presubmit.cfg", - "renovate.json", # to avoid overwriting the ignorePaths list additions: - # ".github/workflows/docs.yml AND lint.yml" specifically - # the version of python referenced in each of those files. - # Currently renovate bot wants to change 3.10 to 3.13. - ], -) - -# ---------------------------------------------------------------------------- -# Fixup files -# ---------------------------------------------------------------------------- - -s.replace( - [".github/header-checker-lint.yml"], - '"Google LLC"', - '"pandas-gbq Authors"', -) - -# ---------------------------------------------------------------------------- -# Samples templates -# ---------------------------------------------------------------------------- - -python.py_samples(skip_readmes=True) - -# ---------------------------------------------------------------------------- -# Final cleanup -# ---------------------------------------------------------------------------- - -s.shell.run(["nox", "-s", "format"], hide_output=False) -for noxfile in REPO_ROOT.glob("samples/**/noxfile.py"): - s.shell.run(["nox", "-s", "format"], cwd=noxfile.parent, hide_output=False) From 5c8b0caf9a9835c34c3b048ff3051e3d9bf7ba87 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Thu, 18 Sep 2025 01:04:47 +0000 Subject: [PATCH 499/519] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot?= =?UTF-8?q?=20post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- packages/pandas-gbq/README.rst | 199 +++++++++++++++--- packages/pandas-gbq/docs/CHANGELOG.md | 1 + packages/pandas-gbq/docs/README.rst | 1 + .../pandas-gbq/pandas-gbq.txt | 0 packages/pandas-gbq/pandas-gbq/pandas-gbq.txt | 0 5 files changed, 169 insertions(+), 32 deletions(-) create mode 120000 packages/pandas-gbq/docs/CHANGELOG.md create mode 120000 packages/pandas-gbq/docs/README.rst rename {owl-bot-staging/pandas-gbq => packages}/pandas-gbq/pandas-gbq.txt (100%) create mode 100644 packages/pandas-gbq/pandas-gbq/pandas-gbq.txt diff --git a/packages/pandas-gbq/README.rst b/packages/pandas-gbq/README.rst index 122f43ee09e8..9fd1759e3456 100644 --- a/packages/pandas-gbq/README.rst +++ b/packages/pandas-gbq/README.rst @@ -1,62 +1,197 @@ -pandas-gbq -========== +Python Client for Google BigQuery connector for pandas +====================================================== |preview| |pypi| |versions| -**pandas-gbq** is a package providing an interface to the Google BigQuery API from pandas. +`Google BigQuery connector for pandas`_: -- `Library Documentation`_ -- `Product Documentation`_ +- `Client Library Documentation`_ +- `Product Documentation`_ .. |preview| image:: https://img.shields.io/badge/support-preview-orange.svg - :target: https://github.com/googleapis/google-cloud-python/blob/main/README.rst#beta-support + :target: https://github.com/googleapis/google-cloud-python/blob/main/README.rst#stability-levels .. |pypi| image:: https://img.shields.io/pypi/v/pandas-gbq.svg :target: https://pypi.org/project/pandas-gbq/ .. |versions| image:: https://img.shields.io/pypi/pyversions/pandas-gbq.svg :target: https://pypi.org/project/pandas-gbq/ -.. _Library Documentation: https://googleapis.dev/python/pandas-gbq/latest/ -.. _Product Documentation: https://cloud.google.com/bigquery/docs/reference/v2/ +.. _Google BigQuery connector for pandas: https://cloud.google.com/bigquery +.. _Client Library Documentation: https://googleapis.dev/python/pandas-gbq/latest/ +.. _Product Documentation: https://cloud.google.com/bigquery + +Quick Start +----------- + +In order to use this library, you first need to go through the following steps: + +1. `Select or create a Cloud Platform project.`_ +2. `Enable billing for your project.`_ +3. `Enable the Google BigQuery connector for pandas.`_ +4. `Set up Authentication.`_ + +.. _Select or create a Cloud Platform project.: https://console.cloud.google.com/project +.. _Enable billing for your project.: https://cloud.google.com/billing/docs/how-to/modify-project#enable_billing_for_a_project +.. _Enable the Google BigQuery connector for pandas.: https://cloud.google.com/bigquery +.. _Set up Authentication.: https://googleapis.dev/python/google-api-core/latest/auth.html Installation ------------- +~~~~~~~~~~~~ -Install latest release version via pip -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Install this library in a virtual environment using `venv`_. `venv`_ is a tool that +creates isolated Python environments. These isolated environments can have separate +versions of Python packages, which allows you to isolate one project's dependencies +from the dependencies of other projects. -.. code-block:: shell +With `venv`_, it's possible to install this library without needing system +install permissions, and without clashing with the installed system +dependencies. - $ pip install pandas-gbq +.. _`venv`: https://docs.python.org/3/library/venv.html -Install latest development version -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. code-block:: shell +Code samples and snippets +~~~~~~~~~~~~~~~~~~~~~~~~~ - $ pip install git+https://github.com/googleapis/python-bigquery-pandas.git +Code samples and snippets live in the `samples/`_ folder. +.. _samples/: https://github.com/googleapis/google-cloud-python/tree/main/packages/pandas-gbq/samples -Usage ------ -Perform a query -~~~~~~~~~~~~~~~ +Supported Python Versions +^^^^^^^^^^^^^^^^^^^^^^^^^ +Our client libraries are compatible with all current `active`_ and `maintenance`_ versions of +Python. -.. code:: python +Python >= 3.7 - import pandas_gbq +.. _active: https://devguide.python.org/devcycle/#in-development-main-branch +.. _maintenance: https://devguide.python.org/devcycle/#maintenance-branches - result_dataframe = pandas_gbq.read_gbq("SELECT column FROM dataset.table WHERE value = 'something'") +Unsupported Python Versions +^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Python <= 3.6 -Upload a dataframe -~~~~~~~~~~~~~~~~~~ +If you are using an `end-of-life`_ +version of Python, we recommend that you update as soon as possible to an actively supported version. -.. code:: python +.. _end-of-life: https://devguide.python.org/devcycle/#end-of-life-branches - import pandas_gbq +Mac/Linux +^^^^^^^^^ - pandas_gbq.to_gbq(dataframe, "dataset.table") +.. code-block:: console -More samples -~~~~~~~~~~~~ + python3 -m venv + source /bin/activate + pip install pandas-gbq + + +Windows +^^^^^^^ + +.. code-block:: console + + py -m venv + .\\Scripts\activate + pip install pandas-gbq + +Next Steps +~~~~~~~~~~ + +- Read the `Client Library Documentation`_ for Google BigQuery connector for pandas + to see other available methods on the client. +- Read the `Google BigQuery connector for pandas Product documentation`_ to learn + more about the product and see How-to Guides. +- View this `README`_ to see the full list of Cloud + APIs that we cover. + +.. _Google BigQuery connector for pandas Product documentation: https://cloud.google.com/bigquery +.. _README: https://github.com/googleapis/google-cloud-python/blob/main/README.rst + +Logging +------- + +This library uses the standard Python :code:`logging` functionality to log some RPC events that could be of interest for debugging and monitoring purposes. +Note the following: + +#. Logs may contain sensitive information. Take care to **restrict access to the logs** if they are saved, whether it be on local storage or on Google Cloud Logging. +#. Google may refine the occurrence, level, and content of various log messages in this library without flagging such changes as breaking. **Do not depend on immutability of the logging events**. +#. By default, the logging events from this library are not handled. You must **explicitly configure log handling** using one of the mechanisms below. + +Simple, environment-based configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To enable logging for this library without any changes in your code, set the :code:`GOOGLE_SDK_PYTHON_LOGGING_SCOPE` environment variable to a valid Google +logging scope. This configures handling of logging events (at level :code:`logging.DEBUG` or higher) from this library in a default manner, emitting the logged +messages in a structured format. It does not currently allow customizing the logging levels captured nor the handlers, formatters, etc. used for any logging +event. + +A logging scope is a period-separated namespace that begins with :code:`google`, identifying the Python module or package to log. + +- Valid logging scopes: :code:`google`, :code:`google.cloud.asset.v1`, :code:`google.api`, :code:`google.auth`, etc. +- Invalid logging scopes: :code:`foo`, :code:`123`, etc. + +**NOTE**: If the logging scope is invalid, the library does not set up any logging handlers. + +Environment-Based Examples +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- Enabling the default handler for all Google-based loggers + +.. code-block:: console + + export GOOGLE_SDK_PYTHON_LOGGING_SCOPE=google + +- Enabling the default handler for a specific Google module (for a client library called :code:`library_v1`): + +.. code-block:: console + + export GOOGLE_SDK_PYTHON_LOGGING_SCOPE=google.cloud.library_v1 + + +Advanced, code-based configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can also configure a valid logging scope using Python's standard `logging` mechanism. + +Code-Based Examples +^^^^^^^^^^^^^^^^^^^ + +- Configuring a handler for all Google-based loggers + +.. code-block:: python + + import logging + + from google.cloud import library_v1 + + base_logger = logging.getLogger("google") + base_logger.addHandler(logging.StreamHandler()) + base_logger.setLevel(logging.DEBUG) + +- Configuring a handler for a specific Google module (for a client library called :code:`library_v1`): + +.. code-block:: python + + import logging + + from google.cloud import library_v1 + + base_logger = logging.getLogger("google.cloud.library_v1") + base_logger.addHandler(logging.StreamHandler()) + base_logger.setLevel(logging.DEBUG) + +Logging details +~~~~~~~~~~~~~~~ + +#. Regardless of which of the mechanisms above you use to configure logging for this library, by default logging events are not propagated up to the root + logger from the `google`-level logger. If you need the events to be propagated to the root logger, you must explicitly set + :code:`logging.getLogger("google").propagate = True` in your code. +#. You can mix the different logging configurations above for different Google modules. For example, you may want use a code-based logging configuration for + one library, but decide you need to also set up environment-based logging configuration for another library. + + #. If you attempt to use both code-based and environment-based configuration for the same module, the environment-based configuration will be ineffectual + if the code -based configuration gets applied first. -See the `pandas-gbq documentation `_ for more details. +#. The Google-specific logging configurations (default handlers for environment-based configuration; not propagating logging events to the root logger) get + executed the first time *any* client library is instantiated in your application, and only if the affected loggers have not been previously configured. + (This is the reason for 2.i. above.) diff --git a/packages/pandas-gbq/docs/CHANGELOG.md b/packages/pandas-gbq/docs/CHANGELOG.md new file mode 120000 index 000000000000..04c99a55caae --- /dev/null +++ b/packages/pandas-gbq/docs/CHANGELOG.md @@ -0,0 +1 @@ +../CHANGELOG.md \ No newline at end of file diff --git a/packages/pandas-gbq/docs/README.rst b/packages/pandas-gbq/docs/README.rst new file mode 120000 index 000000000000..89a0106941ff --- /dev/null +++ b/packages/pandas-gbq/docs/README.rst @@ -0,0 +1 @@ +../README.rst \ No newline at end of file diff --git a/owl-bot-staging/pandas-gbq/pandas-gbq/pandas-gbq.txt b/packages/pandas-gbq/pandas-gbq.txt similarity index 100% rename from owl-bot-staging/pandas-gbq/pandas-gbq/pandas-gbq.txt rename to packages/pandas-gbq/pandas-gbq.txt diff --git a/packages/pandas-gbq/pandas-gbq/pandas-gbq.txt b/packages/pandas-gbq/pandas-gbq/pandas-gbq.txt new file mode 100644 index 000000000000..e69de29bb2d1 From 405851c4b310c96fdccb91f200da8f835dc0fa74 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Thu, 18 Sep 2025 01:04:53 +0000 Subject: [PATCH 500/519] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot?= =?UTF-8?q?=20post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- packages/pandas-gbq/README.rst | 199 +++++++++++++++--- packages/pandas-gbq/docs/CHANGELOG.md | 1 + packages/pandas-gbq/docs/README.rst | 1 + .../pandas-gbq/pandas-gbq.txt | 0 packages/pandas-gbq/pandas-gbq/pandas-gbq.txt | 0 5 files changed, 169 insertions(+), 32 deletions(-) create mode 120000 packages/pandas-gbq/docs/CHANGELOG.md create mode 120000 packages/pandas-gbq/docs/README.rst rename {owl-bot-staging/pandas-gbq => packages}/pandas-gbq/pandas-gbq.txt (100%) create mode 100644 packages/pandas-gbq/pandas-gbq/pandas-gbq.txt diff --git a/packages/pandas-gbq/README.rst b/packages/pandas-gbq/README.rst index 122f43ee09e8..9fd1759e3456 100644 --- a/packages/pandas-gbq/README.rst +++ b/packages/pandas-gbq/README.rst @@ -1,62 +1,197 @@ -pandas-gbq -========== +Python Client for Google BigQuery connector for pandas +====================================================== |preview| |pypi| |versions| -**pandas-gbq** is a package providing an interface to the Google BigQuery API from pandas. +`Google BigQuery connector for pandas`_: -- `Library Documentation`_ -- `Product Documentation`_ +- `Client Library Documentation`_ +- `Product Documentation`_ .. |preview| image:: https://img.shields.io/badge/support-preview-orange.svg - :target: https://github.com/googleapis/google-cloud-python/blob/main/README.rst#beta-support + :target: https://github.com/googleapis/google-cloud-python/blob/main/README.rst#stability-levels .. |pypi| image:: https://img.shields.io/pypi/v/pandas-gbq.svg :target: https://pypi.org/project/pandas-gbq/ .. |versions| image:: https://img.shields.io/pypi/pyversions/pandas-gbq.svg :target: https://pypi.org/project/pandas-gbq/ -.. _Library Documentation: https://googleapis.dev/python/pandas-gbq/latest/ -.. _Product Documentation: https://cloud.google.com/bigquery/docs/reference/v2/ +.. _Google BigQuery connector for pandas: https://cloud.google.com/bigquery +.. _Client Library Documentation: https://googleapis.dev/python/pandas-gbq/latest/ +.. _Product Documentation: https://cloud.google.com/bigquery + +Quick Start +----------- + +In order to use this library, you first need to go through the following steps: + +1. `Select or create a Cloud Platform project.`_ +2. `Enable billing for your project.`_ +3. `Enable the Google BigQuery connector for pandas.`_ +4. `Set up Authentication.`_ + +.. _Select or create a Cloud Platform project.: https://console.cloud.google.com/project +.. _Enable billing for your project.: https://cloud.google.com/billing/docs/how-to/modify-project#enable_billing_for_a_project +.. _Enable the Google BigQuery connector for pandas.: https://cloud.google.com/bigquery +.. _Set up Authentication.: https://googleapis.dev/python/google-api-core/latest/auth.html Installation ------------- +~~~~~~~~~~~~ -Install latest release version via pip -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Install this library in a virtual environment using `venv`_. `venv`_ is a tool that +creates isolated Python environments. These isolated environments can have separate +versions of Python packages, which allows you to isolate one project's dependencies +from the dependencies of other projects. -.. code-block:: shell +With `venv`_, it's possible to install this library without needing system +install permissions, and without clashing with the installed system +dependencies. - $ pip install pandas-gbq +.. _`venv`: https://docs.python.org/3/library/venv.html -Install latest development version -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. code-block:: shell +Code samples and snippets +~~~~~~~~~~~~~~~~~~~~~~~~~ - $ pip install git+https://github.com/googleapis/python-bigquery-pandas.git +Code samples and snippets live in the `samples/`_ folder. +.. _samples/: https://github.com/googleapis/google-cloud-python/tree/main/packages/pandas-gbq/samples -Usage ------ -Perform a query -~~~~~~~~~~~~~~~ +Supported Python Versions +^^^^^^^^^^^^^^^^^^^^^^^^^ +Our client libraries are compatible with all current `active`_ and `maintenance`_ versions of +Python. -.. code:: python +Python >= 3.7 - import pandas_gbq +.. _active: https://devguide.python.org/devcycle/#in-development-main-branch +.. _maintenance: https://devguide.python.org/devcycle/#maintenance-branches - result_dataframe = pandas_gbq.read_gbq("SELECT column FROM dataset.table WHERE value = 'something'") +Unsupported Python Versions +^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Python <= 3.6 -Upload a dataframe -~~~~~~~~~~~~~~~~~~ +If you are using an `end-of-life`_ +version of Python, we recommend that you update as soon as possible to an actively supported version. -.. code:: python +.. _end-of-life: https://devguide.python.org/devcycle/#end-of-life-branches - import pandas_gbq +Mac/Linux +^^^^^^^^^ - pandas_gbq.to_gbq(dataframe, "dataset.table") +.. code-block:: console -More samples -~~~~~~~~~~~~ + python3 -m venv + source /bin/activate + pip install pandas-gbq + + +Windows +^^^^^^^ + +.. code-block:: console + + py -m venv + .\\Scripts\activate + pip install pandas-gbq + +Next Steps +~~~~~~~~~~ + +- Read the `Client Library Documentation`_ for Google BigQuery connector for pandas + to see other available methods on the client. +- Read the `Google BigQuery connector for pandas Product documentation`_ to learn + more about the product and see How-to Guides. +- View this `README`_ to see the full list of Cloud + APIs that we cover. + +.. _Google BigQuery connector for pandas Product documentation: https://cloud.google.com/bigquery +.. _README: https://github.com/googleapis/google-cloud-python/blob/main/README.rst + +Logging +------- + +This library uses the standard Python :code:`logging` functionality to log some RPC events that could be of interest for debugging and monitoring purposes. +Note the following: + +#. Logs may contain sensitive information. Take care to **restrict access to the logs** if they are saved, whether it be on local storage or on Google Cloud Logging. +#. Google may refine the occurrence, level, and content of various log messages in this library without flagging such changes as breaking. **Do not depend on immutability of the logging events**. +#. By default, the logging events from this library are not handled. You must **explicitly configure log handling** using one of the mechanisms below. + +Simple, environment-based configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To enable logging for this library without any changes in your code, set the :code:`GOOGLE_SDK_PYTHON_LOGGING_SCOPE` environment variable to a valid Google +logging scope. This configures handling of logging events (at level :code:`logging.DEBUG` or higher) from this library in a default manner, emitting the logged +messages in a structured format. It does not currently allow customizing the logging levels captured nor the handlers, formatters, etc. used for any logging +event. + +A logging scope is a period-separated namespace that begins with :code:`google`, identifying the Python module or package to log. + +- Valid logging scopes: :code:`google`, :code:`google.cloud.asset.v1`, :code:`google.api`, :code:`google.auth`, etc. +- Invalid logging scopes: :code:`foo`, :code:`123`, etc. + +**NOTE**: If the logging scope is invalid, the library does not set up any logging handlers. + +Environment-Based Examples +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- Enabling the default handler for all Google-based loggers + +.. code-block:: console + + export GOOGLE_SDK_PYTHON_LOGGING_SCOPE=google + +- Enabling the default handler for a specific Google module (for a client library called :code:`library_v1`): + +.. code-block:: console + + export GOOGLE_SDK_PYTHON_LOGGING_SCOPE=google.cloud.library_v1 + + +Advanced, code-based configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can also configure a valid logging scope using Python's standard `logging` mechanism. + +Code-Based Examples +^^^^^^^^^^^^^^^^^^^ + +- Configuring a handler for all Google-based loggers + +.. code-block:: python + + import logging + + from google.cloud import library_v1 + + base_logger = logging.getLogger("google") + base_logger.addHandler(logging.StreamHandler()) + base_logger.setLevel(logging.DEBUG) + +- Configuring a handler for a specific Google module (for a client library called :code:`library_v1`): + +.. code-block:: python + + import logging + + from google.cloud import library_v1 + + base_logger = logging.getLogger("google.cloud.library_v1") + base_logger.addHandler(logging.StreamHandler()) + base_logger.setLevel(logging.DEBUG) + +Logging details +~~~~~~~~~~~~~~~ + +#. Regardless of which of the mechanisms above you use to configure logging for this library, by default logging events are not propagated up to the root + logger from the `google`-level logger. If you need the events to be propagated to the root logger, you must explicitly set + :code:`logging.getLogger("google").propagate = True` in your code. +#. You can mix the different logging configurations above for different Google modules. For example, you may want use a code-based logging configuration for + one library, but decide you need to also set up environment-based logging configuration for another library. + + #. If you attempt to use both code-based and environment-based configuration for the same module, the environment-based configuration will be ineffectual + if the code -based configuration gets applied first. -See the `pandas-gbq documentation `_ for more details. +#. The Google-specific logging configurations (default handlers for environment-based configuration; not propagating logging events to the root logger) get + executed the first time *any* client library is instantiated in your application, and only if the affected loggers have not been previously configured. + (This is the reason for 2.i. above.) diff --git a/packages/pandas-gbq/docs/CHANGELOG.md b/packages/pandas-gbq/docs/CHANGELOG.md new file mode 120000 index 000000000000..04c99a55caae --- /dev/null +++ b/packages/pandas-gbq/docs/CHANGELOG.md @@ -0,0 +1 @@ +../CHANGELOG.md \ No newline at end of file diff --git a/packages/pandas-gbq/docs/README.rst b/packages/pandas-gbq/docs/README.rst new file mode 120000 index 000000000000..89a0106941ff --- /dev/null +++ b/packages/pandas-gbq/docs/README.rst @@ -0,0 +1 @@ +../README.rst \ No newline at end of file diff --git a/owl-bot-staging/pandas-gbq/pandas-gbq/pandas-gbq.txt b/packages/pandas-gbq/pandas-gbq.txt similarity index 100% rename from owl-bot-staging/pandas-gbq/pandas-gbq/pandas-gbq.txt rename to packages/pandas-gbq/pandas-gbq.txt diff --git a/packages/pandas-gbq/pandas-gbq/pandas-gbq.txt b/packages/pandas-gbq/pandas-gbq/pandas-gbq.txt new file mode 100644 index 000000000000..e69de29bb2d1 From b30be8c0c6c444da6d5d6e2d88069ac2878ceace Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 22 Sep 2025 18:22:30 +0000 Subject: [PATCH 501/519] revert README --- packages/pandas-gbq/README.rst | 197 ++++++--------------------------- 1 file changed, 31 insertions(+), 166 deletions(-) diff --git a/packages/pandas-gbq/README.rst b/packages/pandas-gbq/README.rst index 9fd1759e3456..9052a5c975fe 100644 --- a/packages/pandas-gbq/README.rst +++ b/packages/pandas-gbq/README.rst @@ -1,12 +1,12 @@ -Python Client for Google BigQuery connector for pandas -====================================================== +pandas-gbq +========== |preview| |pypi| |versions| -`Google BigQuery connector for pandas`_: +**pandas-gbq** is a package providing an interface to the Google BigQuery API from pandas. -- `Client Library Documentation`_ -- `Product Documentation`_ +- `Library Documentation`_ +- `Product Documentation`_ .. |preview| image:: https://img.shields.io/badge/support-preview-orange.svg :target: https://github.com/googleapis/google-cloud-python/blob/main/README.rst#stability-levels @@ -14,184 +14,49 @@ Python Client for Google BigQuery connector for pandas :target: https://pypi.org/project/pandas-gbq/ .. |versions| image:: https://img.shields.io/pypi/pyversions/pandas-gbq.svg :target: https://pypi.org/project/pandas-gbq/ -.. _Google BigQuery connector for pandas: https://cloud.google.com/bigquery -.. _Client Library Documentation: https://googleapis.dev/python/pandas-gbq/latest/ -.. _Product Documentation: https://cloud.google.com/bigquery - -Quick Start ------------ - -In order to use this library, you first need to go through the following steps: - -1. `Select or create a Cloud Platform project.`_ -2. `Enable billing for your project.`_ -3. `Enable the Google BigQuery connector for pandas.`_ -4. `Set up Authentication.`_ - -.. _Select or create a Cloud Platform project.: https://console.cloud.google.com/project -.. _Enable billing for your project.: https://cloud.google.com/billing/docs/how-to/modify-project#enable_billing_for_a_project -.. _Enable the Google BigQuery connector for pandas.: https://cloud.google.com/bigquery -.. _Set up Authentication.: https://googleapis.dev/python/google-api-core/latest/auth.html +.. _Library Documentation: https://googleapis.dev/python/pandas-gbq/latest/ +.. _Product Documentation: https://cloud.google.com/bigquery/docs/reference/v2/ Installation -~~~~~~~~~~~~ - -Install this library in a virtual environment using `venv`_. `venv`_ is a tool that -creates isolated Python environments. These isolated environments can have separate -versions of Python packages, which allows you to isolate one project's dependencies -from the dependencies of other projects. - -With `venv`_, it's possible to install this library without needing system -install permissions, and without clashing with the installed system -dependencies. - -.. _`venv`: https://docs.python.org/3/library/venv.html - - -Code samples and snippets -~~~~~~~~~~~~~~~~~~~~~~~~~ - -Code samples and snippets live in the `samples/`_ folder. - -.. _samples/: https://github.com/googleapis/google-cloud-python/tree/main/packages/pandas-gbq/samples - - -Supported Python Versions -^^^^^^^^^^^^^^^^^^^^^^^^^ -Our client libraries are compatible with all current `active`_ and `maintenance`_ versions of -Python. - -Python >= 3.7 - -.. _active: https://devguide.python.org/devcycle/#in-development-main-branch -.. _maintenance: https://devguide.python.org/devcycle/#maintenance-branches - -Unsupported Python Versions -^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Python <= 3.6 - -If you are using an `end-of-life`_ -version of Python, we recommend that you update as soon as possible to an actively supported version. - -.. _end-of-life: https://devguide.python.org/devcycle/#end-of-life-branches - -Mac/Linux -^^^^^^^^^ - -.. code-block:: console - - python3 -m venv - source /bin/activate - pip install pandas-gbq - +------------ -Windows -^^^^^^^ +Install latest release version via pip +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. code-block:: console +.. code-block:: shell - py -m venv - .\\Scripts\activate - pip install pandas-gbq + $ pip install pandas-gbq -Next Steps -~~~~~~~~~~ - -- Read the `Client Library Documentation`_ for Google BigQuery connector for pandas - to see other available methods on the client. -- Read the `Google BigQuery connector for pandas Product documentation`_ to learn - more about the product and see How-to Guides. -- View this `README`_ to see the full list of Cloud - APIs that we cover. - -.. _Google BigQuery connector for pandas Product documentation: https://cloud.google.com/bigquery -.. _README: https://github.com/googleapis/google-cloud-python/blob/main/README.rst - -Logging -------- - -This library uses the standard Python :code:`logging` functionality to log some RPC events that could be of interest for debugging and monitoring purposes. -Note the following: - -#. Logs may contain sensitive information. Take care to **restrict access to the logs** if they are saved, whether it be on local storage or on Google Cloud Logging. -#. Google may refine the occurrence, level, and content of various log messages in this library without flagging such changes as breaking. **Do not depend on immutability of the logging events**. -#. By default, the logging events from this library are not handled. You must **explicitly configure log handling** using one of the mechanisms below. - -Simple, environment-based configuration -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To enable logging for this library without any changes in your code, set the :code:`GOOGLE_SDK_PYTHON_LOGGING_SCOPE` environment variable to a valid Google -logging scope. This configures handling of logging events (at level :code:`logging.DEBUG` or higher) from this library in a default manner, emitting the logged -messages in a structured format. It does not currently allow customizing the logging levels captured nor the handlers, formatters, etc. used for any logging -event. - -A logging scope is a period-separated namespace that begins with :code:`google`, identifying the Python module or package to log. - -- Valid logging scopes: :code:`google`, :code:`google.cloud.asset.v1`, :code:`google.api`, :code:`google.auth`, etc. -- Invalid logging scopes: :code:`foo`, :code:`123`, etc. - -**NOTE**: If the logging scope is invalid, the library does not set up any logging handlers. - -Environment-Based Examples -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -- Enabling the default handler for all Google-based loggers - -.. code-block:: console - - export GOOGLE_SDK_PYTHON_LOGGING_SCOPE=google - -- Enabling the default handler for a specific Google module (for a client library called :code:`library_v1`): - -.. code-block:: console - - export GOOGLE_SDK_PYTHON_LOGGING_SCOPE=google.cloud.library_v1 +Install latest development version +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. code-block:: shell -Advanced, code-based configuration -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + $ pip install "pandas-gbq @ git+https://github.com/googleapis/google-cloud-python#egg=pandas-gbq&subdirectory=packages/pandas-gbq" -You can also configure a valid logging scope using Python's standard `logging` mechanism. -Code-Based Examples -^^^^^^^^^^^^^^^^^^^ +Usage +----- -- Configuring a handler for all Google-based loggers +Perform a query +~~~~~~~~~~~~~~~ -.. code-block:: python +.. code:: python - import logging - - from google.cloud import library_v1 - - base_logger = logging.getLogger("google") - base_logger.addHandler(logging.StreamHandler()) - base_logger.setLevel(logging.DEBUG) + import pandas_gbq -- Configuring a handler for a specific Google module (for a client library called :code:`library_v1`): + result_dataframe = pandas_gbq.read_gbq("SELECT column FROM dataset.table WHERE value = 'something'") -.. code-block:: python +Upload a dataframe +~~~~~~~~~~~~~~~~~~ - import logging - - from google.cloud import library_v1 - - base_logger = logging.getLogger("google.cloud.library_v1") - base_logger.addHandler(logging.StreamHandler()) - base_logger.setLevel(logging.DEBUG) +.. code:: python -Logging details -~~~~~~~~~~~~~~~ + import pandas_gbq -#. Regardless of which of the mechanisms above you use to configure logging for this library, by default logging events are not propagated up to the root - logger from the `google`-level logger. If you need the events to be propagated to the root logger, you must explicitly set - :code:`logging.getLogger("google").propagate = True` in your code. -#. You can mix the different logging configurations above for different Google modules. For example, you may want use a code-based logging configuration for - one library, but decide you need to also set up environment-based logging configuration for another library. + pandas_gbq.to_gbq(dataframe, "dataset.table") - #. If you attempt to use both code-based and environment-based configuration for the same module, the environment-based configuration will be ineffectual - if the code -based configuration gets applied first. +More samples +~~~~~~~~~~~~ -#. The Google-specific logging configurations (default handlers for environment-based configuration; not propagating logging events to the root logger) get - executed the first time *any* client library is instantiated in your application, and only if the affected loggers have not been previously configured. - (This is the reason for 2.i. above.) +See the `pandas-gbq documentation `_ for more details. From b8688df09412b83f2ba25b21395221a6289bee03 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 22 Sep 2025 18:33:03 +0000 Subject: [PATCH 502/519] fix docs --- packages/pandas-gbq/docs/README.rst | 1 - packages/pandas-gbq/docs/index.rst | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 120000 packages/pandas-gbq/docs/README.rst diff --git a/packages/pandas-gbq/docs/README.rst b/packages/pandas-gbq/docs/README.rst deleted file mode 120000 index 89a0106941ff..000000000000 --- a/packages/pandas-gbq/docs/README.rst +++ /dev/null @@ -1 +0,0 @@ -../README.rst \ No newline at end of file diff --git a/packages/pandas-gbq/docs/index.rst b/packages/pandas-gbq/docs/index.rst index 14fcde286ee2..f06a1e9cf68f 100644 --- a/packages/pandas-gbq/docs/index.rst +++ b/packages/pandas-gbq/docs/index.rst @@ -49,7 +49,7 @@ Contents: writing.rst api.rst contributing.rst - changelog.md + CHANGELOG.md privacy.rst oauth.rst From 384befba0281f94c59a221e6b2e4c8edd3fe0277 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 22 Sep 2025 18:36:31 +0000 Subject: [PATCH 503/519] update default python version to 3.13 --- packages/pandas-gbq/noxfile.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index e2e9f7234e97..179d30736a6a 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -32,7 +32,17 @@ ISORT_VERSION = "isort==5.10.1" LINT_PATHS = ["docs", "pandas_gbq", "tests", "noxfile.py", "setup.py"] -DEFAULT_PYTHON_VERSION = "3.10" +ALL_PYTHON = [ + "3.7", + "3.8", + "3.9", + "3.10", + "3.11", + "3.12", + "3.13", +] + +DEFAULT_PYTHON_VERSION = ALL_PYTHON[-1] UNIT_TEST_PYTHON_VERSIONS = ["3.9", "3.10", "3.11", "3.12", "3.13"] UNIT_TEST_STANDARD_DEPENDENCIES = [ From 2f3c3e1f452e1fb833f2350df6d3eea12e8de7af Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 22 Sep 2025 18:43:57 +0000 Subject: [PATCH 504/519] fix lint check --- packages/pandas-gbq/noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 179d30736a6a..6dca8b9b3582 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -171,7 +171,7 @@ def format(session): @_calculate_duration def lint_setup_py(session): """Verify that setup.py is valid (including RST check).""" - session.install("docutils", "pygments") + session.install("setuptools", "docutils", "pygments") session.run("python", "setup.py", "check", "--restructuredtext", "--strict") From 9a60ba7c6b12142acbcd6f845a20bfe0d9ec8437 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 22 Sep 2025 18:49:47 +0000 Subject: [PATCH 505/519] rename prerelease to prerelease_deps --- packages/pandas-gbq/noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 6dca8b9b3582..fcc83dea96aa 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -318,7 +318,7 @@ def system(session): @nox.session(python=DEFAULT_PYTHON_VERSION) @_calculate_duration -def prerelease(session): +def prerelease_deps(session): session.install( # https://arrow.apache.org/docs/developers/python.html#installing-nightly-packages "--extra-index-url", From 1f154ceb7a52c2f0788ace7be4bb4e10dac12163 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 22 Sep 2025 19:08:38 +0000 Subject: [PATCH 506/519] skip Python runtimes which are not in scope for unit testing --- packages/pandas-gbq/noxfile.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index fcc83dea96aa..416546aad358 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -230,10 +230,12 @@ def default(session): ) -@nox.session(python=UNIT_TEST_PYTHON_VERSIONS) +@nox.session(python=ALL_PYTHON) @_calculate_duration def unit(session): """Run the unit test suite.""" + if session.python not in UNIT_TEST_PYTHON_VERSIONS: + session.skip(f"Testing of Python runtime {session.python} skipped because it's not in {UNIT_TEST_PYTHON_VERSIONS}") default(session) From 6d1eb61b13df5ff286e6b002ccac3008136b18ad Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 22 Sep 2025 19:16:14 +0000 Subject: [PATCH 507/519] lint --- packages/pandas-gbq/noxfile.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 416546aad358..fe1891de0911 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -235,7 +235,9 @@ def default(session): def unit(session): """Run the unit test suite.""" if session.python not in UNIT_TEST_PYTHON_VERSIONS: - session.skip(f"Testing of Python runtime {session.python} skipped because it's not in {UNIT_TEST_PYTHON_VERSIONS}") + session.skip( + f"Testing of Python runtime {session.python} skipped because it's not in {UNIT_TEST_PYTHON_VERSIONS}" + ) default(session) From 3d40de0cb91109786faebadd40f94c28d48f8b30 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 22 Sep 2025 20:45:00 +0000 Subject: [PATCH 508/519] add core_deps_from_source nox session --- packages/pandas-gbq/noxfile.py | 97 ++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index fe1891de0911..f611e5deb6c8 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -410,6 +410,103 @@ def prerelease_deps(session): ) +@nox.session(python=DEFAULT_PYTHON_VERSION) +@_calculate_duration +def core_deps_from_source(session): + session.install( + # https://arrow.apache.org/docs/developers/python.html#installing-nightly-packages + "--extra-index-url", + "https://pypi.fury.io/arrow-nightlies/", + "--prefer-binary", + "--pre", + "--upgrade", + "pyarrow", + ) + session.install( + "--prefer-binary", + "--pre", + "--upgrade", + "google-cloud-core", + "google-resumable-media", + # Exclude version 1.49.0rc1 which has a known issue. See https://github.com/grpc/grpc/pull/30642 + "grpcio!=1.49.0rc1", + "pandas", + ) + session.install( + "freezegun", + "google-cloud-datacatalog", + "google-cloud-storage", + "google-cloud-testutils", + "IPython", + "mock", + "psutil", + "pytest", + "pytest-cov", + ) + + # Install python-bigquery from main to detect + # any potential breaking changes. For context, see: + # https://github.com/googleapis/python-bigquery-pandas/issues/854 + session.install("https://github.com/googleapis/python-bigquery/archive/main.zip") + + # Because we test minimum dependency versions on the minimum Python + # version, the first version we test with in the unit tests sessions has a + # constraints file containing all dependencies and extras. + with open( + CURRENT_DIRECTORY + / "testing" + / f"constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt", + encoding="utf-8", + ) as constraints_file: + constraints_text = constraints_file.read() + + # Ignore leading whitespace and comment lines. + constraints_deps = [ + match.group(1) + for match in re.finditer( + r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE + ) + ] + + # We use --no-deps to ensure that pre-release versions aren't overwritten + # by the version ranges in setup.py. + session.install(*constraints_deps) + session.install("--no-deps", "-e", ".[all]") + + # TODO(https://github.com/googleapis/gapic-generator-python/issues/2358): `grpcio` and + # `grpcio-status` should be added to the list below so that they are installed from source, + # rather than PyPI. + # TODO(https://github.com/googleapis/gapic-generator-python/issues/2357): `protobuf` should be + # added to the list below so that it is installed from source, rather than PyPI + # Note: If a dependency is added to the `core_dependencies_from_source` list, + # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. + core_dependencies_from_source = [ + "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", + "google-api-core @ git+https://github.com/googleapis/python-api-core.git", + "google-auth @ git+https://github.com/googleapis/google-auth-library-python.git", + "google-cloud-bigquery-storage @ git+https://github.com/googleapis/google-cloud-python.git@main#subdirectory=packages/google-cloud-bigquery-storage", + "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", + "proto-plus @ git+https://github.com/googleapis/proto-plus-python.git", + ] + + for dep in core_dependencies_from_source: + session.install(dep, "--no-deps", "--ignore-installed") + print(f"Installed {dep}") + + # Print out prerelease package versions. + session.run("python", "-m", "pip", "freeze") + + # Run all tests, except a few samples tests which require extra dependencies. + session.run( + "py.test", + "--quiet", + "-W default::PendingDeprecationWarning", + f"--junitxml=prerelease_unit_{session.python}_sponge_log.xml", + os.path.join("tests", "unit"), + *session.posargs, + ) + + @nox.session(python=DEFAULT_PYTHON_VERSION) @_calculate_duration def cover(session): From deeb83c45a300e231c71f4498a7cdf9efc6d954c Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 23 Sep 2025 16:48:19 +0000 Subject: [PATCH 509/519] skip system tests when running in github actions --- packages/pandas-gbq/noxfile.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index f611e5deb6c8..4d3e18ff6f93 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -400,14 +400,16 @@ def prerelease_deps(session): *session.posargs, ) - session.run( - "py.test", - "--quiet", - "-W default::PendingDeprecationWarning", - f"--junitxml=prerelease_system_{session.python}_sponge_log.xml", - os.path.join("tests", "system"), - *session.posargs, - ) + # Skip system tests when running in Github Actions since there are no credentials + if os.environ.get("GITHUB_ACTIONS", "false") == "true": + session.run( + "py.test", + "--quiet", + "-W default::PendingDeprecationWarning", + f"--junitxml=prerelease_system_{session.python}_sponge_log.xml", + os.path.join("tests", "system"), + *session.posargs, + ) @nox.session(python=DEFAULT_PYTHON_VERSION) From c04f4d2503e58b870f12ac661c613db6b55e5f46 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 23 Sep 2025 17:22:17 +0000 Subject: [PATCH 510/519] skip system tests when running in github actions --- packages/pandas-gbq/noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 4d3e18ff6f93..35f8f8a60a3b 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -401,7 +401,7 @@ def prerelease_deps(session): ) # Skip system tests when running in Github Actions since there are no credentials - if os.environ.get("GITHUB_ACTIONS", "false") == "true": + if os.environ.get("GITHUB_ACTIONS", "false") == "false": session.run( "py.test", "--quiet", From c3a32e2662006e7eb4e46de0f7de56cef1f5a75d Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 23 Sep 2025 17:22:46 +0000 Subject: [PATCH 511/519] wip --- .librarian/state.yaml | 332 ++++++++++-------- .../packages_to_onboard.yaml | 8 +- 2 files changed, 179 insertions(+), 161 deletions(-) diff --git a/.librarian/state.yaml b/.librarian/state.yaml index b375dc441f09..6fb42088d48b 100644 --- a/.librarian/state.yaml +++ b/.librarian/state.yaml @@ -1,156 +1,180 @@ image: us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/python-librarian-generator:latest libraries: - - id: google-cloud-dlp - version: 3.32.0 - last_generated_commit: f8776fec04e336527ba7279d960105533a1c4e21 - apis: - - path: google/privacy/dlp/v2 - service_config: dlp_v2.yaml - source_roots: - - packages/google-cloud-dlp - preserve_regex: - - packages/google-cloud-dlp/CHANGELOG.md - - docs/CHANGELOG.md - - docs/README.rst - - samples/README.txt - - tar.gz - - gapic_version.py - - samples/generated_samples/snippet_metadata_ - - scripts/client-post-processing - - samples/snippets/README.rst - - tests/system - remove_regex: - - packages/google-cloud-dlp - tag_format: '{id}-v{version}' - - id: google-cloud-eventarc - version: 1.15.3 - last_generated_commit: d300b151a973ce0425ae4ad07b3de957ca31bec6 - apis: - - path: google/cloud/eventarc/v1 - source_roots: - - packages/google-cloud-eventarc - preserve_regex: - - packages/google-cloud-eventarc/CHANGELOG.md - - docs/CHANGELOG.md - - docs/README.rst - - samples/README.txt - - tar.gz - - gapic_version.py - - samples/generated_samples/snippet_metadata_ - - scripts/client-post-processing - - samples/snippets/README.rst - - tests/system - remove_regex: - - packages/google-cloud-eventarc - tag_format: '{id}-v{version}' - - id: google-cloud-video-live-stream - version: 1.12.0 - last_generated_commit: d300b151a973ce0425ae4ad07b3de957ca31bec6 - apis: - - path: google/cloud/video/livestream/v1 - source_roots: - - packages/google-cloud-video-live-stream - preserve_regex: - - packages/google-cloud-video-live-stream/CHANGELOG.md - - docs/CHANGELOG.md - - docs/README.rst - - samples/README.txt - - tar.gz - - gapic_version.py - - samples/generated_samples/snippet_metadata_ - - scripts/client-post-processing - - samples/snippets/README.rst - - tests/system - remove_regex: - - packages/google-cloud-video-live-stream - tag_format: '{id}-v{version}' - - id: google-ads-marketingplatform-admin - version: 0.1.6 - last_generated_commit: d300b151a973ce0425ae4ad07b3de957ca31bec6 - apis: - - path: google/marketingplatform/admin/v1alpha - source_roots: - - packages/google-ads-marketingplatform-admin - preserve_regex: - - packages/google-ads-marketingplatform-admin/CHANGELOG.md - - docs/CHANGELOG.md - - docs/README.rst - - samples/README.txt - - tar.gz - - gapic_version.py - - samples/generated_samples/snippet_metadata_ - - scripts/client-post-processing - - samples/snippets/README.rst - - tests/system - remove_regex: - - packages/google-ads-marketingplatform-admin - tag_format: '{id}-v{version}' - - id: google-ai-generativelanguage - version: 0.7.0 - last_generated_commit: d300b151a973ce0425ae4ad07b3de957ca31bec6 - apis: - - path: google/ai/generativelanguage/v1 - - path: google/ai/generativelanguage/v1beta - - path: google/ai/generativelanguage/v1beta3 - - path: google/ai/generativelanguage/v1beta2 - - path: google/ai/generativelanguage/v1alpha - source_roots: - - packages/google-ai-generativelanguage - preserve_regex: - - packages/google-ai-generativelanguage/CHANGELOG.md - - docs/CHANGELOG.md - - docs/README.rst - - samples/README.txt - - tar.gz - - gapic_version.py - - samples/generated_samples/snippet_metadata_ - - scripts/client-post-processing - - samples/snippets/README.rst - - tests/system - remove_regex: - - packages/google-ai-generativelanguage - tag_format: '{id}-v{version}' - - id: google-analytics-admin - version: 0.25.0 - last_generated_commit: d300b151a973ce0425ae4ad07b3de957ca31bec6 - apis: - - path: google/analytics/admin/v1beta - - path: google/analytics/admin/v1alpha - source_roots: - - packages/google-analytics-admin - preserve_regex: - - packages/google-analytics-admin/CHANGELOG.md - - docs/CHANGELOG.md - - docs/README.rst - - samples/README.txt - - tar.gz - - gapic_version.py - - samples/generated_samples/snippet_metadata_ - - scripts/client-post-processing - - samples/snippets/README.rst - - tests/system - remove_regex: - - packages/google-analytics-admin - tag_format: '{id}-v{version}' - - id: google-analytics-data - version: 0.18.19 - last_generated_commit: d300b151a973ce0425ae4ad07b3de957ca31bec6 - apis: - - path: google/analytics/data/v1alpha - - path: google/analytics/data/v1beta - source_roots: - - packages/google-analytics-data - preserve_regex: - - packages/google-analytics-data/CHANGELOG.md - - docs/CHANGELOG.md - - docs/README.rst - - samples/README.txt - - tar.gz - - gapic_version.py - - samples/generated_samples/snippet_metadata_ - - scripts/client-post-processing - - samples/snippets/README.rst - - tests/system - remove_regex: - - packages/google-analytics-data - tag_format: '{id}-v{version}' +- id: google-cloud-dlp + version: 3.32.0 + last_generated_commit: f8776fec04e336527ba7279d960105533a1c4e21 + apis: + - path: google/privacy/dlp/v2 + service_config: dlp_v2.yaml + source_roots: + - packages/google-cloud-dlp + preserve_regex: + - packages/google-cloud-dlp/CHANGELOG.md + - docs/CHANGELOG.md + - docs/README.rst + - samples/README.txt + - tar.gz + - gapic_version.py + - samples/generated_samples/snippet_metadata_ + - scripts/client-post-processing + - samples/snippets/README.rst + - tests/system + remove_regex: + - packages/google-cloud-dlp + tag_format: '{id}-v{version}' +- id: google-cloud-eventarc + version: 1.15.3 + last_generated_commit: d300b151a973ce0425ae4ad07b3de957ca31bec6 + apis: + - path: google/cloud/eventarc/v1 + source_roots: + - packages/google-cloud-eventarc + preserve_regex: + - packages/google-cloud-eventarc/CHANGELOG.md + - docs/CHANGELOG.md + - docs/README.rst + - samples/README.txt + - tar.gz + - gapic_version.py + - samples/generated_samples/snippet_metadata_ + - scripts/client-post-processing + - samples/snippets/README.rst + - tests/system + remove_regex: + - packages/google-cloud-eventarc + tag_format: '{id}-v{version}' +- id: google-cloud-video-live-stream + version: 1.12.0 + last_generated_commit: d300b151a973ce0425ae4ad07b3de957ca31bec6 + apis: + - path: google/cloud/video/livestream/v1 + source_roots: + - packages/google-cloud-video-live-stream + preserve_regex: + - packages/google-cloud-video-live-stream/CHANGELOG.md + - docs/CHANGELOG.md + - docs/README.rst + - samples/README.txt + - tar.gz + - gapic_version.py + - samples/generated_samples/snippet_metadata_ + - scripts/client-post-processing + - samples/snippets/README.rst + - tests/system + remove_regex: + - packages/google-cloud-video-live-stream + tag_format: '{id}-v{version}' +- id: google-ads-marketingplatform-admin + version: 0.1.6 + last_generated_commit: d300b151a973ce0425ae4ad07b3de957ca31bec6 + apis: + - path: google/marketingplatform/admin/v1alpha + source_roots: + - packages/google-ads-marketingplatform-admin + preserve_regex: + - packages/google-ads-marketingplatform-admin/CHANGELOG.md + - docs/CHANGELOG.md + - docs/README.rst + - samples/README.txt + - tar.gz + - gapic_version.py + - samples/generated_samples/snippet_metadata_ + - scripts/client-post-processing + - samples/snippets/README.rst + - tests/system + remove_regex: + - packages/google-ads-marketingplatform-admin + tag_format: '{id}-v{version}' +- id: google-ai-generativelanguage + version: 0.7.0 + last_generated_commit: d300b151a973ce0425ae4ad07b3de957ca31bec6 + apis: + - path: google/ai/generativelanguage/v1 + - path: google/ai/generativelanguage/v1beta + - path: google/ai/generativelanguage/v1beta3 + - path: google/ai/generativelanguage/v1beta2 + - path: google/ai/generativelanguage/v1alpha + source_roots: + - packages/google-ai-generativelanguage + preserve_regex: + - packages/google-ai-generativelanguage/CHANGELOG.md + - docs/CHANGELOG.md + - docs/README.rst + - samples/README.txt + - tar.gz + - gapic_version.py + - samples/generated_samples/snippet_metadata_ + - scripts/client-post-processing + - samples/snippets/README.rst + - tests/system + remove_regex: + - packages/google-ai-generativelanguage + tag_format: '{id}-v{version}' +- id: google-analytics-admin + version: 0.25.0 + last_generated_commit: d300b151a973ce0425ae4ad07b3de957ca31bec6 + apis: + - path: google/analytics/admin/v1beta + - path: google/analytics/admin/v1alpha + source_roots: + - packages/google-analytics-admin + preserve_regex: + - packages/google-analytics-admin/CHANGELOG.md + - docs/CHANGELOG.md + - docs/README.rst + - samples/README.txt + - tar.gz + - gapic_version.py + - samples/generated_samples/snippet_metadata_ + - scripts/client-post-processing + - samples/snippets/README.rst + - tests/system + remove_regex: + - packages/google-analytics-admin + tag_format: '{id}-v{version}' +- id: google-analytics-data + version: 0.18.19 + last_generated_commit: d300b151a973ce0425ae4ad07b3de957ca31bec6 + apis: + - path: google/analytics/data/v1alpha + - path: google/analytics/data/v1beta + source_roots: + - packages/google-analytics-data + preserve_regex: + - packages/google-analytics-data/CHANGELOG.md + - docs/CHANGELOG.md + - docs/README.rst + - samples/README.txt + - tar.gz + - gapic_version.py + - samples/generated_samples/snippet_metadata_ + - scripts/client-post-processing + - samples/snippets/README.rst + - tests/system + remove_regex: + - packages/google-analytics-data + tag_format: '{id}-v{version}' +- id: google-cloud-bigquery-storage + version: 2.33.1 + last_generated_commit: d300b151a973ce0425ae4ad07b3de957ca31bec6 + apis: + - path: google/cloud/bigquery/storage/v1beta2 + - path: google/cloud/bigquery/storage/v1alpha + - path: google/cloud/bigquery/storage/v1beta + - path: google/cloud/bigquery/storage/v1 + source_roots: + - packages/google-cloud-bigquery-storage + preserve_regex: + - packages/google-cloud-bigquery-storage/CHANGELOG.md + - docs/CHANGELOG.md + - docs/README.rst + - samples/README.txt + - tar.gz + - gapic_version.py + - samples/generated_samples/snippet_metadata_ + - scripts/client-post-processing + - samples/snippets/README.rst + - tests/system + remove_regex: + - packages/google-cloud-bigquery-storage + tag_format: '{id}-v{version}' diff --git a/scripts/configure_state_yaml/packages_to_onboard.yaml b/scripts/configure_state_yaml/packages_to_onboard.yaml index 8e19ef17952c..fa797f2894b5 100644 --- a/scripts/configure_state_yaml/packages_to_onboard.yaml +++ b/scripts/configure_state_yaml/packages_to_onboard.yaml @@ -14,11 +14,5 @@ # packages_to_onboard: [ - "google-ads-marketingplatform-admin", - "google-ai-generativelanguage", - "google-analytics-admin", - "google-analytics-data", - "google-cloud-dlp", - "google-cloud-eventarc", - "google-cloud-video-live-stream", + "pandas-gbq", ] From 230e78da12d2a54fd9a63536c58efb632ec3167c Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 23 Sep 2025 17:34:30 +0000 Subject: [PATCH 512/519] onboard to librarian --- .librarian/state.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.librarian/state.yaml b/.librarian/state.yaml index 6fb42088d48b..394aef237f6f 100644 --- a/.librarian/state.yaml +++ b/.librarian/state.yaml @@ -1,4 +1,6 @@ image: us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/python-librarian-generator:latest +libraries: + image: us-central1-docker.pkg.dev/cloud-sdk-librarian-prod/images-prod/python-librarian-generator:latest libraries: - id: google-cloud-dlp version: 3.32.0 @@ -178,3 +180,12 @@ libraries: remove_regex: - packages/google-cloud-bigquery-storage tag_format: '{id}-v{version}' +- id: pandas-gbq + version: 0.29.2 + last_generated_commit: d300b151a973ce0425ae4ad07b3de957ca31bec6 + apis: [] + source_roots: + - packages/pandas-gbq + preserve_regex: [] + remove_regex: [] + tag_format: '{id}-v{version}' From 4b530f807e5eec4a7878bef2f0a0941e24e35b6e Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 23 Sep 2025 17:34:42 +0000 Subject: [PATCH 513/519] clean up obsolete files --- packages/pandas-gbq/.OwlBot.yaml | 17 ------ packages/pandas-gbq/.gitattributes | 1 - packages/pandas-gbq/.gitignore | 64 --------------------- packages/pandas-gbq/.pre-commit-config.yaml | 31 ---------- packages/pandas-gbq/SECURITY.md | 7 --- packages/pandas-gbq/pandas-gbq.txt | 0 packages/pandas-gbq/release-procedure.md | 38 ------------ packages/pandas-gbq/renovate.json | 12 ---- 8 files changed, 170 deletions(-) delete mode 100644 packages/pandas-gbq/.OwlBot.yaml delete mode 100644 packages/pandas-gbq/.gitattributes delete mode 100644 packages/pandas-gbq/.gitignore delete mode 100644 packages/pandas-gbq/.pre-commit-config.yaml delete mode 100644 packages/pandas-gbq/SECURITY.md delete mode 100644 packages/pandas-gbq/pandas-gbq.txt delete mode 100644 packages/pandas-gbq/release-procedure.md delete mode 100644 packages/pandas-gbq/renovate.json diff --git a/packages/pandas-gbq/.OwlBot.yaml b/packages/pandas-gbq/.OwlBot.yaml deleted file mode 100644 index 68b0f07b1488..000000000000 --- a/packages/pandas-gbq/.OwlBot.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -begin-after-commit-hash: 1afeb53252641dc35a421fa5acc59e2f3229ad6d - diff --git a/packages/pandas-gbq/.gitattributes b/packages/pandas-gbq/.gitattributes deleted file mode 100644 index 9afa48862e72..000000000000 --- a/packages/pandas-gbq/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -pandas_gbq/_version.py export-subst diff --git a/packages/pandas-gbq/.gitignore b/packages/pandas-gbq/.gitignore deleted file mode 100644 index d083ea1ddc3e..000000000000 --- a/packages/pandas-gbq/.gitignore +++ /dev/null @@ -1,64 +0,0 @@ -*.py[cod] -*.sw[op] - -# C extensions -*.so - -# Packages -*.egg -*.egg-info -dist -build -eggs -.eggs -parts -bin -var -sdist -develop-eggs -.installed.cfg -lib -lib64 -__pycache__ - -# Installer logs -pip-log.txt - -# Unit test / coverage reports -.coverage -.nox -.cache -.pytest_cache - - -# Mac -.DS_Store - -# JetBrains -.idea - -# VS Code -.vscode - -# emacs -*~ - -# Built documentation -docs/_build -bigquery/docs/generated -docs.metadata - -# Virtual environment -env/ -venv/ - -# Test logs -coverage.xml -*sponge_log.xml - -# System test environment variables. -system_tests/local_test_setup - -# Make sure a generated file isn't accidentally committed. -pylintrc -pylintrc.test diff --git a/packages/pandas-gbq/.pre-commit-config.yaml b/packages/pandas-gbq/.pre-commit-config.yaml deleted file mode 100644 index 1d74695f70b6..000000000000 --- a/packages/pandas-gbq/.pre-commit-config.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# See https://pre-commit.com for more information -# See https://pre-commit.com/hooks.html for more hooks -repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.0.1 - hooks: - - id: trailing-whitespace - - id: end-of-file-fixer - - id: check-yaml -- repo: https://github.com/psf/black - rev: 23.7.0 - hooks: - - id: black -- repo: https://github.com/pycqa/flake8 - rev: 6.1.0 - hooks: - - id: flake8 diff --git a/packages/pandas-gbq/SECURITY.md b/packages/pandas-gbq/SECURITY.md deleted file mode 100644 index 8b58ae9c01ae..000000000000 --- a/packages/pandas-gbq/SECURITY.md +++ /dev/null @@ -1,7 +0,0 @@ -# Security Policy - -To report a security issue, please use [g.co/vulnz](https://g.co/vulnz). - -The Google Security Team will respond within 5 working days of your report on g.co/vulnz. - -We use g.co/vulnz for our intake, and do coordination and disclosure here using GitHub Security Advisory to privately discuss and fix the issue. diff --git a/packages/pandas-gbq/pandas-gbq.txt b/packages/pandas-gbq/pandas-gbq.txt deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/packages/pandas-gbq/release-procedure.md b/packages/pandas-gbq/release-procedure.md deleted file mode 100644 index 47c8fd74fe7a..000000000000 --- a/packages/pandas-gbq/release-procedure.md +++ /dev/null @@ -1,38 +0,0 @@ - -* Send PR to prepare release on scheduled date. - -* Verify your local repository is on the latest changes. `rebase -i` should be noop. - - git fetch pandas-gbq master - git checkout master - git rebase -i pandas-gbq/master - git diff pandas-gbq/master - -* Tag commit - - git tag -a x.x.x -m 'Version x.x.x' - -* Push to GitHub - - git push pandas-gbq master --tags - -* Build the package - - git clean -xfd - python setup.py register sdist bdist_wheel - -* Upload to test PyPI - - twine upload --repository testpypi dist/* - -* Try out test PyPI package - - pip install --upgrade --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple pandas-gbq - -* Upload to PyPI - - twine upload dist/* - -* Create the [release on GitHub](https://github.com/googleapis/python-bigquery-pandas/releases/new) using the tag created earlier. - - * Upload wheel and source zip from `dist/` directory. diff --git a/packages/pandas-gbq/renovate.json b/packages/pandas-gbq/renovate.json deleted file mode 100644 index 9d9a6d0bb8cc..000000000000 --- a/packages/pandas-gbq/renovate.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "extends": [ - "config:base", - "group:all", - ":preserveSemverRanges", - ":disableDependencyDashboard" - ], - "ignorePaths": [".pre-commit-config.yaml", ".kokoro/requirements.txt", "setup.py", ".github/workflows/unittest.yml", ".github/workflows/docs.yml", ".github/workflows/lint.yml"], - "pip_requirements": { - "fileMatch": ["requirements-test.txt", "samples/[\\S/]*constraints.txt", "samples/[\\S/]*constraints-test.txt"] - } -} From 63931e3a509f8af2c5a13cd51148962e48672c06 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 23 Sep 2025 17:41:53 +0000 Subject: [PATCH 514/519] add link to PyPI history --- packages/pandas-gbq/CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index 1cd1478cbceb..2813dff5af42 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +[PyPI History][1] + +[1]: https://pypi.org/project/pandas-gbq/#history + ## [0.29.2](https://github.com/googleapis/python-bigquery-pandas/compare/v0.29.1...v0.29.2) (2025-07-10) From eb1f2241253288e081864fcccd2afc1dc6d9e699 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 23 Sep 2025 17:43:00 +0000 Subject: [PATCH 515/519] remove obsolete file --- packages/pandas-gbq/CODE_OF_CONDUCT.md | 95 -------------------------- 1 file changed, 95 deletions(-) delete mode 100644 packages/pandas-gbq/CODE_OF_CONDUCT.md diff --git a/packages/pandas-gbq/CODE_OF_CONDUCT.md b/packages/pandas-gbq/CODE_OF_CONDUCT.md deleted file mode 100644 index 039f43681204..000000000000 --- a/packages/pandas-gbq/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,95 +0,0 @@ - -# Code of Conduct - -## Our Pledge - -In the interest of fostering an open and welcoming environment, we as -contributors and maintainers pledge to making participation in our project and -our community a harassment-free experience for everyone, regardless of age, body -size, disability, ethnicity, gender identity and expression, level of -experience, education, socio-economic status, nationality, personal appearance, -race, religion, or sexual identity and orientation. - -## Our Standards - -Examples of behavior that contributes to creating a positive environment -include: - -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members - -Examples of unacceptable behavior by participants include: - -* The use of sexualized language or imagery and unwelcome sexual attention or - advances -* Trolling, insulting/derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or electronic - address, without explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting - -## Our Responsibilities - -Project maintainers are responsible for clarifying the standards of acceptable -behavior and are expected to take appropriate and fair corrective action in -response to any instances of unacceptable behavior. - -Project maintainers have the right and responsibility to remove, edit, or reject -comments, commits, code, wiki edits, issues, and other contributions that are -not aligned to this Code of Conduct, or to ban temporarily or permanently any -contributor for other behaviors that they deem inappropriate, threatening, -offensive, or harmful. - -## Scope - -This Code of Conduct applies both within project spaces and in public spaces -when an individual is representing the project or its community. Examples of -representing a project or community include using an official project e-mail -address, posting via an official social media account, or acting as an appointed -representative at an online or offline event. Representation of a project may be -further defined and clarified by project maintainers. - -This Code of Conduct also applies outside the project spaces when the Project -Steward has a reasonable belief that an individual's behavior may have a -negative impact on the project or its community. - -## Conflict Resolution - -We do not believe that all conflict is bad; healthy debate and disagreement -often yield positive results. However, it is never okay to be disrespectful or -to engage in behavior that violates the project’s code of conduct. - -If you see someone violating the code of conduct, you are encouraged to address -the behavior directly with those involved. Many issues can be resolved quickly -and easily, and this gives people more control over the outcome of their -dispute. If you are unable to resolve the matter for any reason, or if the -behavior is threatening or harassing, report it. We are dedicated to providing -an environment where participants feel welcome and safe. - - -Reports should be directed to *googleapis-stewards@google.com*, the -Project Steward(s) for *Google Cloud Client Libraries*. It is the Project Steward’s duty to -receive and address reported violations of the code of conduct. They will then -work with a committee consisting of representatives from the Open Source -Programs Office and the Google Open Source Strategy team. If for any reason you -are uncomfortable reaching out to the Project Steward, please email -opensource@google.com. - -We will investigate every complaint, but you may not receive a direct response. -We will use our discretion in determining when and how to follow up on reported -incidents, which may range from not taking action to permanent expulsion from -the project and project-sponsored spaces. We will notify the accused of the -report and provide them an opportunity to discuss it before any action is taken. -The identity of the reporter will be omitted from the details of the report -supplied to the accused. In potentially harmful situations, such as ongoing -harassment or threats to anyone's safety, we may take action without notice. - -## Attribution - -This Code of Conduct is adapted from the Contributor Covenant, version 1.4, -available at -https://www.contributor-covenant.org/version/1/4/code-of-conduct.html \ No newline at end of file From bfd5c0f37f67fa8b955753f6f936a6ab726b90e6 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 23 Sep 2025 18:33:27 +0000 Subject: [PATCH 516/519] clean up --- packages/pandas-gbq/pandas-gbq/pandas-gbq.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 packages/pandas-gbq/pandas-gbq/pandas-gbq.txt diff --git a/packages/pandas-gbq/pandas-gbq/pandas-gbq.txt b/packages/pandas-gbq/pandas-gbq/pandas-gbq.txt deleted file mode 100644 index e69de29bb2d1..000000000000 From acae1980bc808bfbe13664753f1bec57b1d3ff6b Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 23 Sep 2025 18:36:46 +0000 Subject: [PATCH 517/519] Update comment --- packages/pandas-gbq/noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 35f8f8a60a3b..64b3b93eba5e 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -498,7 +498,7 @@ def core_deps_from_source(session): # Print out prerelease package versions. session.run("python", "-m", "pip", "freeze") - # Run all tests, except a few samples tests which require extra dependencies. + # Run unit tests only session.run( "py.test", "--quiet", From 0c6a4634ad5dbd59e7f6a5d9954ae95f056d80af Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 23 Sep 2025 18:51:05 +0000 Subject: [PATCH 518/519] add a placeholder for running system tests --- packages/pandas-gbq/noxfile.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 64b3b93eba5e..583c2bbdcbb5 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -498,16 +498,26 @@ def core_deps_from_source(session): # Print out prerelease package versions. session.run("python", "-m", "pip", "freeze") - # Run unit tests only + # Run unit tests session.run( "py.test", "--quiet", "-W default::PendingDeprecationWarning", - f"--junitxml=prerelease_unit_{session.python}_sponge_log.xml", + f"--junitxml=core_deps_from_source_unit_{session.python}_sponge_log.xml", os.path.join("tests", "unit"), *session.posargs, ) + # Skip system tests when running in Github Actions since there are no credentials + if os.environ.get("GITHUB_ACTIONS", "false") == "false": + session.run( + "py.test", + "--quiet", + "-W default::PendingDeprecationWarning", + f"--junitxml=core_deps_from_source_system_{session.python}_sponge_log.xml", + os.path.join("tests", "system"), + *session.posargs, + ) @nox.session(python=DEFAULT_PYTHON_VERSION) @_calculate_duration From dc1ef661b3ce97b95727a9ae260a828539a43978 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 23 Sep 2025 18:53:17 +0000 Subject: [PATCH 519/519] lint --- packages/pandas-gbq/noxfile.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 583c2bbdcbb5..db2b965f28cd 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -519,6 +519,7 @@ def core_deps_from_source(session): *session.posargs, ) + @nox.session(python=DEFAULT_PYTHON_VERSION) @_calculate_duration def cover(session):