From 6dec022975e8f6721cbc80a5a8c34e30ad9152ed Mon Sep 17 00:00:00 2001 From: Jens Reidel Date: Sat, 4 Jul 2020 18:43:40 +0200 Subject: [PATCH 1/2] Update CI builds --- .ci/build-manylinux-wheels.sh | 11 +- .ci/travis-before-install.sh | 2 +- .ci/travis-build-wheels.sh | 19 ++- .ci/travis-release.sh | 18 +-- .travis.yml | 216 +++++++++++++++++++++------------- asyncpg/__init__.py | 2 +- asyncpg/connect_utils.py | 13 +- asyncpg/connection.py | 9 +- asyncpg/pgproto | 2 +- setup.py | 15 ++- tests/test__environment.py | 2 + tests/test_connect.py | 38 ++++++ tests/test_copy.py | 43 +++++++ 13 files changed, 279 insertions(+), 111 deletions(-) diff --git a/.ci/build-manylinux-wheels.sh b/.ci/build-manylinux-wheels.sh index b5dc579b..40030785 100755 --- a/.ci/build-manylinux-wheels.sh +++ b/.ci/build-manylinux-wheels.sh @@ -2,22 +2,25 @@ set -e -x +# iproute isn't included in CentOS 7 +yum install -y iproute + # Compile wheels PYTHON="/opt/python/${PYTHON_VERSION}/bin/python" PIP="/opt/python/${PYTHON_VERSION}/bin/pip" -${PIP} install --upgrade setuptools pip wheel~=0.31.1 +${PIP} install --upgrade setuptools pip wheel cd /io make clean ${PYTHON} setup.py bdist_wheel # Bundle external shared libraries into the wheels. for whl in /io/dist/*.whl; do - auditwheel repair $whl -w /io/dist/ + auditwheel repair $whl -w /tmp/ + ${PIP} install /tmp/*.whl + mv /tmp/*.whl /io/dist/ rm /io/dist/*-linux_*.whl done -${PIP} install ${PYMODULE} -f "file:///io/dist" - # Grab docker host, where Postgres should be running. export PGHOST=$(ip route | awk '/default/ { print $3 }' | uniq) export PGUSER="postgres" diff --git a/.ci/travis-before-install.sh b/.ci/travis-before-install.sh index 3dd6b8a8..dd624ffa 100755 --- a/.ci/travis-before-install.sh +++ b/.ci/travis-before-install.sh @@ -27,7 +27,7 @@ if [[ "${TRAVIS_OS_NAME}" == "linux" && "${BUILD}" == *wheels* ]]; then fi fi - sudo service postgresql start ${PGVERSION} + sudo pg_ctlcluster ${PGVERSION} main restart fi if [ "${TRAVIS_OS_NAME}" == "osx" ]; then diff --git a/.ci/travis-build-wheels.sh b/.ci/travis-build-wheels.sh index 6249190c..11445af2 100755 --- a/.ci/travis-build-wheels.sh +++ b/.ci/travis-build-wheels.sh @@ -3,7 +3,7 @@ set -e -x -if [[ "${TRAVIS_BRANCH}" != "releases" || "${BUILD}" != *wheels* ]]; then +if [[ "${TRAVIS_BRANCH}" != "release" || "${BUILD}" != *wheels* ]]; then # Not a release exit 0 fi @@ -45,9 +45,16 @@ if [ "${TRAVIS_OS_NAME}" == "linux" ]; then s='m' if tuple('${pyver}'.split('.')) < ('3', '8') \ else ''))") + if [[ "$(uname -m)" = "x86_64" ]]; then + ARCHES="x86_64 i686" + MANYLINUX_VERSION="1" + elif [[ "$(uname -m)" = "aarch64" ]]; then + ARCHES="aarch64" + MANYLINUX_VERSION="2014" + fi - for arch in x86_64 i686; do - ML_IMAGE="quay.io/pypa/manylinux1_${arch}" + for arch in $ARCHES; do + ML_IMAGE="quay.io/pypa/manylinux${MANYLINUX_VERSION}_${arch}" docker pull "${ML_IMAGE}" docker run --rm \ -v "${_root}":/io \ @@ -64,9 +71,11 @@ elif [ "${TRAVIS_OS_NAME}" == "osx" ]; then export PGINSTALLATION="/usr/local/opt/postgresql@${PGVERSION}/bin" make clean - python setup.py bdist_wheel + python setup.py bdist_wheel --dist-dir /tmp/ - pip install ${PYMODULE} -f "file:///${_root}/dist" + pip install /tmp/*.whl + mkdir -p "${_root}/dist" + mv /tmp/*.whl "${_root}/dist/" make -C "${_root}" ASYNCPG_VERSION="${PACKAGE_VERSION}" testinstalled _upload_wheels diff --git a/.ci/travis-release.sh b/.ci/travis-release.sh index 4b6a999a..3840c620 100755 --- a/.ci/travis-release.sh +++ b/.ci/travis-release.sh @@ -18,13 +18,17 @@ if [ "${PACKAGE_VERSION}" == "${PYPI_VERSION}" ]; then fi # Check if all expected wheels have been built and uploaded. -release_platforms=( - "macosx_10_??_x86_64" - "manylinux1_i686" - "manylinux1_x86_64" - "win32" - "win_amd64" -) +if [[ "$(uname -m)" = "x86_64" ]]; then + release_platforms=( + "macosx_10_??_x86_64" + "manylinux1_i686" + "manylinux1_x86_64" + "win32" + "win_amd64" + ) +elif [[ "$(uname -m)" = "aarch64" ]]; then + release_platforms="manylinux2014_aarch64" +fi P="${PYMODULE}-${PACKAGE_VERSION}" expected_wheels=() diff --git a/.travis.yml b/.travis.yml index be029141..680f91f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,12 +18,6 @@ env: - DOCS_PUSH_KEY_LABEL=0760b951e99c -addons: - apt: - sources: - - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ trusty-pgdg main' - key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' - branches: # Avoid building PR branches. only: @@ -32,132 +26,190 @@ branches: - releases - /^v\d+(\.\d+)*$/ -matrix: +jobs: fast_finish: true include: # Do quick test runs for each supported version of PostgreSQL # minus the latest. - - os: linux - dist: xenial - language: python - python: "3.6" - env: BUILD=quicktests PGVERSION=9.2 - addons: - apt: {packages: [postgresql-9.2, postgresql-contrib-9.2]} - - - os: linux - dist: xenial - language: python - python: "3.6" - env: BUILD=quicktests PGVERSION=9.3 - addons: - apt: {packages: [postgresql-9.3, postgresql-contrib-9.3]} - - - os: linux - dist: xenial + - name: "Quicktest psql 9.5" + os: linux + dist: focal language: python - python: "3.6" - env: BUILD=quicktests PGVERSION=9.4 - addons: - apt: {packages: [postgresql-9.4, postgresql-contrib-9.4]} - - - os: linux - dist: xenial - language: python - python: "3.6" + python: "3.8" env: BUILD=quicktests PGVERSION=9.5 addons: - apt: {packages: [postgresql-9.5, postgresql-contrib-9.5]} - - - os: linux - dist: xenial + apt: + sources: + - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' + key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' + packages: + - postgresql-9.5 + - postgresql-contrib-9.5 + + - name: "Quicktest psql 9.6" + os: linux + dist: focal language: python - python: "3.6" + python: "3.8" env: BUILD=quicktests PGVERSION=9.6 addons: - apt: {packages: [postgresql-9.6, postgresql-contrib-9.6]} - - - os: linux - dist: xenial + apt: + sources: + - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' + key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' + packages: + - postgresql-9.6 + - postgresql-contrib-9.6 + + - name: "Quicktest psql 10" + os: linux + dist: focal language: python - python: "3.6" + python: "3.8" env: BUILD=quicktests PGVERSION=10 addons: - apt: {packages: [postgresql-10]} - - - os: linux - dist: xenial + apt: + sources: + - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' + key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' + packages: + - postgresql-10 + + - name: "Quicktest psql 11" + os: linux + dist: focal language: python - python: "3.6" + python: "3.8" env: BUILD=quicktests PGVERSION=11 addons: - apt: {packages: [postgresql-11]} + apt: + sources: + - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' + key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' + packages: + - postgresql-11 # Do a full test run on the latest supported version of PostgreSQL # on each supported version of Python. - - os: linux - dist: xenial - sudo: required + - name: "Test py 3.5" + os: linux + dist: focal language: python python: "3.5" env: BUILD=tests PGVERSION=12 addons: - apt: {packages: [postgresql-12]} - - - os: linux - dist: xenial - sudo: required + apt: + sources: + - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' + key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' + packages: + - postgresql-12 + + - name: "Test py 3.6" + os: linux + dist: focal language: python python: "3.6" env: BUILD=tests PGVERSION=12 addons: - apt: {packages: [postgresql-12]} - - - os: linux - dist: xenial - sudo: true + apt: + sources: + - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' + key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' + packages: + - postgresql-12 + + - name: "Test py 3.7" + os: linux + dist: focal language: python python: "3.7" env: BUILD=tests PGVERSION=12 addons: - apt: {packages: [postgresql-12]} - - - os: linux - dist: xenial - sudo: true + apt: + sources: + - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' + key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' + packages: + - postgresql-12 + + - name: "Test py 3.8" + os: linux + dist: focal language: python python: "3.8" env: BUILD=tests PGVERSION=12 addons: - apt: {packages: [postgresql-12]} + apt: + sources: + - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' + key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' + packages: + - postgresql-12 + + # Only test on recent aarch64 distribution + # 3.7 is the latest supported by Travis + # https://docs.travis-ci.com/user/languages/python/#python-versions + # The shipped Postgres 9.X collides with the 12 on aarch64 + # until fixed, use official ubuntu repos + - name: "Test aarch64 py 3.8-dev" + os: linux + arch: arm64 + dist: focal + language: python + python: "3.8-dev" + env: BUILD=tests PGVERSION=12 + addons: + postgresql: "12" # Build manylinux wheels. Each wheel will be tested, # so there is no need for BUILD=tests here. # Also use this job to publish the releases and build # the documentation. - - os: linux - dist: xenial - sudo: required + - name: "x86 wheels and docs" + os: linux + dist: focal language: python - python: "3.6" + python: "3.8" env: BUILD=wheels,docs,release PGVERSION=12 services: [docker] addons: - apt: {packages: [postgresql-12]} + apt: + sources: + - sourceline: 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' + key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' + packages: + - postgresql-12 + + # Same for the aarch64 manylinux wheel + - name: "aarch64 wheels" + os: linux + arch: arm64 + dist: focal + language: python + python: "3.8-dev" + env: BUILD=wheels,release PGVERSION=12 + services: [docker] + addons: + postgresql: "12" - - os: osx - env: BUILD=tests,wheels PYTHON_VERSION=3.5.7 PGVERSION=10 + - name: "OSX py 3.5" + os: osx + env: BUILD=tests,wheels PYTHON_VERSION=3.5.9 PGVERSION=12 - - os: osx - env: BUILD=tests,wheels PYTHON_VERSION=3.6.9 PGVERSION=10 + - name: "OSX py 3.6" + os: osx + env: BUILD=tests,wheels PYTHON_VERSION=3.6.10 PGVERSION=12 - - os: osx - env: BUILD=tests,wheels PYTHON_VERSION=3.7.4 PGVERSION=10 + - name: "OSX py 3.7" + os: osx + env: BUILD=tests,wheels PYTHON_VERSION=3.7.7 PGVERSION=12 - - os: osx - env: BUILD=tests,wheels PYTHON_VERSION=3.8.0 PGVERSION=10 + - name: "OSX py 3.8" + os: osx + env: BUILD=tests,wheels PYTHON_VERSION=3.8.3 PGVERSION=12 cache: pip diff --git a/asyncpg/__init__.py b/asyncpg/__init__.py index ee66f932..b0cef053 100644 --- a/asyncpg/__init__.py +++ b/asyncpg/__init__.py @@ -31,4 +31,4 @@ # snapshots will automatically include the git revision # in __version__, for example: '0.16.0.dev0+ge06ad03' -__version__ = '0.20.0' +__version__ = '0.21.0.dev0' diff --git a/asyncpg/connect_utils.py b/asyncpg/connect_utils.py index ec3d1090..2678b358 100644 --- a/asyncpg/connect_utils.py +++ b/asyncpg/connect_utils.py @@ -21,6 +21,7 @@ import typing import urllib.parse import warnings +import inspect from . import compat from . import exceptions @@ -601,6 +602,16 @@ async def _connect_addr(*, addr, loop, timeout, params, config, raise asyncio.TimeoutError connected = _create_future(loop) + + params_input = params + if callable(params.password): + if inspect.iscoroutinefunction(params.password): + password = await params.password() + else: + password = params.password() + + params = params._replace(password=password) + proto_factory = lambda: protocol.Protocol( addr, connected, params, loop) @@ -633,7 +644,7 @@ async def _connect_addr(*, addr, loop, timeout, params, config, tr.close() raise - con = connection_class(pr, tr, loop, addr, config, params) + con = connection_class(pr, tr, loop, addr, config, params_input) pr.set_connection(con) return con diff --git a/asyncpg/connection.py b/asyncpg/connection.py index ef1b595d..5fa2ddae 100644 --- a/asyncpg/connection.py +++ b/asyncpg/connection.py @@ -813,7 +813,7 @@ async def _copy_in(self, copy_stmt, source, timeout): if path is not None: # a path - f = await run_in_executor(None, open, path, 'wb') + f = await run_in_executor(None, open, path, 'rb') opened_by_us = True elif hasattr(source, 'read'): # file-like @@ -1566,6 +1566,10 @@ async def connect(dsn=None, *, other users and applications may be able to read it without needing specific privileges. It is recommended to use *passfile* instead. + Password may be either a string, or a callable that returns a string. + If a callable is provided, it will be called each time a new connection + is established. + :param passfile: The name of the file used to store passwords (defaults to ``~/.pgpass``, or ``%APPDATA%\postgresql\pgpass.conf`` @@ -1646,6 +1650,9 @@ async def connect(dsn=None, *, Added ability to specify multiple hosts in the *dsn* and *host* arguments. + .. versionchanged:: 0.21.0 + The *password* argument now accepts a callable or an async function. + .. _SSLContext: https://docs.python.org/3/library/ssl.html#ssl.SSLContext .. _create_default_context: https://docs.python.org/3/library/ssl.html#ssl.create_default_context diff --git a/asyncpg/pgproto b/asyncpg/pgproto index 1b4af3c6..76091445 160000 --- a/asyncpg/pgproto +++ b/asyncpg/pgproto @@ -1 +1 @@ -Subproject commit 1b4af3c685bfda6c74c082504fe8947c0a55e358 +Subproject commit 76091445db8b49a7d78504b47eb34fcbfbb89567 diff --git a/setup.py b/setup.py index 5e2e1494..5bb9e693 100644 --- a/setup.py +++ b/setup.py @@ -19,15 +19,13 @@ # We use vanilla build_ext, to avoid importing Cython via # the setuptools version. -from distutils import extension as distutils_extension -from distutils.command import build_ext as distutils_build_ext - import setuptools from setuptools.command import build_py as setuptools_build_py from setuptools.command import sdist as setuptools_sdist +from setuptools.command import build_ext as setuptools_build_ext -CYTHON_DEPENDENCY = 'Cython==0.29.14' +CYTHON_DEPENDENCY = 'Cython==0.29.20' # Minimal dependencies required to test asyncpg. TEST_DEPENDENCIES = [ @@ -138,9 +136,9 @@ def build_module(self, module, module_file, package): return outfile, copied -class build_ext(distutils_build_ext.build_ext): +class build_ext(setuptools_build_ext.build_ext): - user_options = distutils_build_ext.build_ext.user_options + [ + user_options = setuptools_build_ext.build_ext.user_options + [ ('cython-always', None, 'run cythonize() even if .c files are present'), ('cython-annotate', None, @@ -264,6 +262,7 @@ def finalize_options(self): 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: Implementation :: CPython', 'Topic :: Database :: Front-Ends', ], @@ -278,13 +277,13 @@ def finalize_options(self): provides=['asyncpg'], include_package_data=True, ext_modules=[ - distutils_extension.Extension( + setuptools.extension.Extension( "asyncpg.pgproto.pgproto", ["asyncpg/pgproto/pgproto.pyx"], extra_compile_args=CFLAGS, extra_link_args=LDFLAGS), - distutils_extension.Extension( + setuptools.extension.Extension( "asyncpg.protocol.protocol", ["asyncpg/protocol/record/recordobj.c", "asyncpg/protocol/protocol.pyx"], diff --git a/tests/test__environment.py b/tests/test__environment.py index 878fb2fc..3141fe8b 100644 --- a/tests/test__environment.py +++ b/tests/test__environment.py @@ -30,6 +30,8 @@ async def test_environment_server_version(self): @unittest.skipIf(not os.environ.get('ASYNCPG_VERSION'), "environ[ASYNCPG_VERSION] is not set") + @unittest.skipIf("dev" in asyncpg.__version__, + "development version with git commit data") async def test_environment_asyncpg_version(self): apgver = os.environ.get('ASYNCPG_VERSION') self.assertEqual( diff --git a/tests/test_connect.py b/tests/test_connect.py index f0767827..116b8ad9 100644 --- a/tests/test_connect.py +++ b/tests/test_connect.py @@ -204,6 +204,44 @@ async def test_auth_password_cleartext(self): user='password_user', password='wrongpassword') + async def test_auth_password_cleartext_callable(self): + def get_correctpassword(): + return 'correctpassword' + + def get_wrongpassword(): + return 'wrongpassword' + + conn = await self.connect( + user='password_user', + password=get_correctpassword) + await conn.close() + + with self.assertRaisesRegex( + asyncpg.InvalidPasswordError, + 'password authentication failed for user "password_user"'): + await self._try_connect( + user='password_user', + password=get_wrongpassword) + + async def test_auth_password_cleartext_callable_coroutine(self): + async def get_correctpassword(): + return 'correctpassword' + + async def get_wrongpassword(): + return 'wrongpassword' + + conn = await self.connect( + user='password_user', + password=get_correctpassword) + await conn.close() + + with self.assertRaisesRegex( + asyncpg.InvalidPasswordError, + 'password authentication failed for user "password_user"'): + await self._try_connect( + user='password_user', + password=get_wrongpassword) + async def test_auth_password_md5(self): conn = await self.connect( user='md5_user', password='correctpassword') diff --git a/tests/test_copy.py b/tests/test_copy.py index 257cc79c..dd01153f 100644 --- a/tests/test_copy.py +++ b/tests/test_copy.py @@ -8,6 +8,7 @@ import asyncio import datetime import io +import os import tempfile import asyncpg @@ -582,6 +583,48 @@ async def __anext__(self): finally: await self.con.execute('DROP TABLE copytab') + async def test_copy_to_table_from_file_path(self): + await self.con.execute(''' + CREATE TABLE copytab(a text, "b~" text, i int); + ''') + + f = tempfile.NamedTemporaryFile(delete=False) + try: + f.write( + '\n'.join([ + 'a1\tb1\t1', + 'a2\tb2\t2', + 'a3\tb3\t3', + 'a4\tb4\t4', + 'a5\tb5\t5', + '*\t\\N\t\\N', + '' + ]).encode('utf-8') + ) + f.close() + + res = await self.con.copy_to_table('copytab', source=f.name) + self.assertEqual(res, 'COPY 6') + + output = await self.con.fetch(""" + SELECT * FROM copytab ORDER BY a + """) + self.assertEqual( + output, + [ + ('*', None, None), + ('a1', 'b1', 1), + ('a2', 'b2', 2), + ('a3', 'b3', 3), + ('a4', 'b4', 4), + ('a5', 'b5', 5), + ] + ) + + finally: + await self.con.execute('DROP TABLE public.copytab') + os.unlink(f.name) + async def test_copy_records_to_table_1(self): await self.con.execute(''' CREATE TABLE copytab(a text, b int, c timestamptz); From e2b800d7def8e9375172465e78a67ffa8dd28fcc Mon Sep 17 00:00:00 2001 From: Jens Reidel Date: Sat, 4 Jul 2020 18:44:53 +0200 Subject: [PATCH 2/2] Fix branch name --- .ci/travis-build-wheels.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/travis-build-wheels.sh b/.ci/travis-build-wheels.sh index 11445af2..ae8c52af 100755 --- a/.ci/travis-build-wheels.sh +++ b/.ci/travis-build-wheels.sh @@ -3,7 +3,7 @@ set -e -x -if [[ "${TRAVIS_BRANCH}" != "release" || "${BUILD}" != *wheels* ]]; then +if [[ "${TRAVIS_BRANCH}" != "releases" || "${BUILD}" != *wheels* ]]; then # Not a release exit 0 fi pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy