From a382b3d6286178e6a3eeb78a48621f9bcc75ad8c Mon Sep 17 00:00:00 2001 From: Georgy Frolov Date: Mon, 4 Nov 2019 13:29:36 +0300 Subject: [PATCH 1/8] try more default socket paths --- mycli/main.py | 4 ++-- mycli/packages/filepaths.py | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 7f42360c..9085e655 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -49,7 +49,7 @@ from .lexer import MyCliLexer from .__init__ import __version__ from .compat import WIN -from .packages.filepaths import dir_path_exists +from .packages.filepaths import dir_path_exists, guess_socket_location import itertools @@ -429,7 +429,7 @@ def _connect(): # Try a sensible default socket first (simplifies auth) # If we get a connection error, try tcp/ip localhost try: - socket = '/var/run/mysqld/mysqld.sock' + socket = guess_socket_location() _connect() except OperationalError as e: # These are "Can't open socket" and 2x "Can't connect" diff --git a/mycli/packages/filepaths.py b/mycli/packages/filepaths.py index 5ebdcd97..f3de46e1 100644 --- a/mycli/packages/filepaths.py +++ b/mycli/packages/filepaths.py @@ -3,11 +3,13 @@ from mycli.encodingutils import text_type import os +DEFAULT_SOCKET_DIRS = ('/var/run/', '/var/lib/', '/tmp') + def list_path(root_dir): """List directory if exists. - :param dir: str + :param root_dir: str :return: list """ @@ -84,3 +86,15 @@ def dir_path_exists(path): """ return os.path.exists(os.path.dirname(path)) + + +def guess_socket_location(): + """Try to guess the location of the default mysql socket file.""" + socket_dirs = filter(os.path.exists, DEFAULT_SOCKET_DIRS) + for directory in socket_dirs: + for r, dirs, files in os.walk(directory, topdown=True): + for filename in files: + if filename.startswith('mysql') and filename.endswith('.socket'): + return os.path.join(r, filename) + dirs[:] = [d for d in dirs if d.startswith('mysql')] + return '' From ec17f50b3067516ab203c935d9d66b54911d81d2 Mon Sep 17 00:00:00 2001 From: Georgy Frolov Date: Tue, 3 Mar 2020 06:57:56 +0300 Subject: [PATCH 2/8] platform-specific default socket paths --- mycli/main.py | 6 ++++++ mycli/packages/filepaths.py | 16 ++++++++++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 9085e655..fe9c3eff 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -9,6 +9,7 @@ import re import fileinput from collections import namedtuple +from pwd import getpwuid from time import time from datetime import datetime from random import choice @@ -451,6 +452,11 @@ def _connect(): _connect() else: raise e + else: + socket_owner = getpwuid(os.stat(socket).st_uid).pw_name + self.echo( + "Using socket {}, owned by user {}".format(socket, socket_owner) + ) else: host = host or 'localhost' port = port or 3306 diff --git a/mycli/packages/filepaths.py b/mycli/packages/filepaths.py index f3de46e1..ef309e62 100644 --- a/mycli/packages/filepaths.py +++ b/mycli/packages/filepaths.py @@ -2,8 +2,16 @@ from __future__ import unicode_literals from mycli.encodingutils import text_type import os +import platform -DEFAULT_SOCKET_DIRS = ('/var/run/', '/var/lib/', '/tmp') + +if os.name == "posix": + if platform.system() == "Darwin": + DEFAULT_SOCKET_DIRS = ("/tmp",) + else: + DEFAULT_SOCKET_DIRS = ("/var/run", "/var/lib") +else: + DEFAULT_SOCKET_DIRS = () def list_path(root_dir): @@ -94,7 +102,7 @@ def guess_socket_location(): for directory in socket_dirs: for r, dirs, files in os.walk(directory, topdown=True): for filename in files: - if filename.startswith('mysql') and filename.endswith('.socket'): + if filename.startswith("mysql") and filename.endswith(".socket"): return os.path.join(r, filename) - dirs[:] = [d for d in dirs if d.startswith('mysql')] - return '' + dirs[:] = [d for d in dirs if d.startswith("mysql")] + return "" From dc5c14ee1f6af5794661d4768471d77318502029 Mon Sep 17 00:00:00 2001 From: Georgy Frolov Date: Tue, 3 Mar 2020 07:38:20 +0300 Subject: [PATCH 3/8] use socket from my.cnf if available --- mycli/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index fe9c3eff..bac58d85 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -426,11 +426,11 @@ def _connect(): raise e try: - if (socket is host is port is None) and not WIN: + if (host is None) and not WIN: # Try a sensible default socket first (simplifies auth) # If we get a connection error, try tcp/ip localhost try: - socket = guess_socket_location() + socket = socket or guess_socket_location() _connect() except OperationalError as e: # These are "Can't open socket" and 2x "Can't connect" From 3db74ddadcc53f59fc2787464420046b81f145c2 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Sat, 18 Apr 2020 19:50:28 -0700 Subject: [PATCH 4/8] Lint fixes. --- mycli/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index bac58d85..248aa9f6 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -455,7 +455,8 @@ def _connect(): else: socket_owner = getpwuid(os.stat(socket).st_uid).pw_name self.echo( - "Using socket {}, owned by user {}".format(socket, socket_owner) + "Using socket {}, owned by user {}".format( + socket, socket_owner) ) else: host = host or 'localhost' From a15df7416b40a1872e374f3c75ab19f3fc78d984 Mon Sep 17 00:00:00 2001 From: Georgy Frolov Date: Thu, 23 Apr 2020 12:46:09 +0300 Subject: [PATCH 5/8] fixed sillent tcp/ip fallbacks inside pymysql --- mycli/main.py | 23 +++++++++++++++-------- mycli/packages/filepaths.py | 5 +++-- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/mycli/main.py b/mycli/main.py index 4f9c3bdb..1c01d2aa 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -430,6 +430,17 @@ def _connect(): else: raise e + def _fallback_to_tcp_ip(): + self.echo( + 'Retrying over TCP/IP', err=True) + + # Else fall back to TCP/IP localhost + nonlocal socket, host, port + socket = "" + host = 'localhost' + port = 3306 + _connect() + try: if (host is None) and not WIN: # Try a sensible default socket first (simplifies auth) @@ -437,6 +448,9 @@ def _connect(): try: socket = socket or guess_socket_location() _connect() + except FileNotFoundError: + self.echo('Failed to find socket file at default locations') + _fallback_to_tcp_ip() except OperationalError as e: # These are "Can't open socket" and 2x "Can't connect" if [code for code in (2001, 2002, 2003) if code == e.args[0]]: @@ -447,14 +461,7 @@ def _connect(): self.echo( "Failed to connect to local MySQL server through socket '{}':".format(socket)) self.echo(str(e), err=True) - self.echo( - 'Retrying over TCP/IP', err=True) - - # Else fall back to TCP/IP localhost - socket = "" - host = 'localhost' - port = 3306 - _connect() + _fallback_to_tcp_ip() else: raise e else: diff --git a/mycli/packages/filepaths.py b/mycli/packages/filepaths.py index f2950d7d..26b77c40 100644 --- a/mycli/packages/filepaths.py +++ b/mycli/packages/filepaths.py @@ -99,7 +99,8 @@ def guess_socket_location(): for directory in socket_dirs: for r, dirs, files in os.walk(directory, topdown=True): for filename in files: - if filename.startswith("mysql") and filename.endswith(".socket"): + name, ext = os.path.splitext(filename) + if name.startswith("mysql") and ext in ('.socket', '.sock'): return os.path.join(r, filename) dirs[:] = [d for d in dirs if d.startswith("mysql")] - return "" + raise FileNotFoundError From df44b068ae10dfa37eac4682c62ddff932966cc3 Mon Sep 17 00:00:00 2001 From: Georgy Frolov Date: Wed, 6 May 2020 19:16:17 +0300 Subject: [PATCH 6/8] fix discovering .cnf files that are included with !includedirs --- mycli/config.py | 27 ++++++++++- mycli/main.py | 97 ++++++++++++++++++++++++++----------- mycli/packages/filepaths.py | 2 +- mycli/sqlexecute.py | 1 - 4 files changed, 96 insertions(+), 31 deletions(-) diff --git a/mycli/config.py b/mycli/config.py index 77475099..03fb502a 100644 --- a/mycli/config.py +++ b/mycli/config.py @@ -1,4 +1,5 @@ import shutil +from copy import copy from io import BytesIO, TextIOWrapper import logging import os @@ -58,12 +59,34 @@ def read_config_file(f, list_values=True): return config +def get_included_configs(config_path) -> list: + """Get a list of configuration files that are included into config_path + with !includedir directive.""" + if not os.path.exists(config_path): + return [] + included_configs = [] + with open(config_path) as f: + include_directives = filter( + lambda s: s.startswith('!includedir'), + f + ) + dirs = map(lambda s: s.strip().split()[-1], include_directives) + dirs = filter(os.path.isdir, dirs) + for dir in dirs: + for filename in os.listdir(dir): + if filename.endswith('.cnf'): + included_configs.append(os.path.join(dir, filename)) + return included_configs + + def read_config_files(files, list_values=True): """Read and merge a list of config files.""" config = ConfigObj(list_values=list_values) - - for _file in files: + _files = copy(files) + while _files: + _file = _files.pop(0) + _files = get_included_configs(_file) + _files _config = read_config_file(_file, list_values=list_values) if bool(_config) is True: config.merge(_config) diff --git a/mycli/main.py b/mycli/main.py index 1c01d2aa..55afedd5 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -386,7 +386,7 @@ def connect(self, database='', user='', passwd='', host='', port='', if port or host: socket = '' else: - socket = socket or cnf['socket'] + socket = socket or cnf['socket'] or guess_socket_location() user = user or cnf['user'] or os.getenv('USER') host = host or cnf['host'] port = port or cnf['port'] @@ -430,27 +430,13 @@ def _connect(): else: raise e - def _fallback_to_tcp_ip(): - self.echo( - 'Retrying over TCP/IP', err=True) - - # Else fall back to TCP/IP localhost - nonlocal socket, host, port - socket = "" - host = 'localhost' - port = 3306 - _connect() - try: - if (host is None) and not WIN: - # Try a sensible default socket first (simplifies auth) - # If we get a connection error, try tcp/ip localhost + if not WIN and socket: + socket_owner = getpwuid(os.stat(socket).st_uid).pw_name + self.echo( + f"Connecting to socket {socket}, owned by user {socket_owner}") try: - socket = socket or guess_socket_location() _connect() - except FileNotFoundError: - self.echo('Failed to find socket file at default locations') - _fallback_to_tcp_ip() except OperationalError as e: # These are "Can't open socket" and 2x "Can't connect" if [code for code in (2001, 2002, 2003) if code == e.args[0]]: @@ -461,15 +447,16 @@ def _fallback_to_tcp_ip(): self.echo( "Failed to connect to local MySQL server through socket '{}':".format(socket)) self.echo(str(e), err=True) - _fallback_to_tcp_ip() + self.echo( + 'Retrying over TCP/IP', err=True) + + # Else fall back to TCP/IP localhost + socket = "" + host = 'localhost' + port = 3306 + _connect() else: raise e - else: - socket_owner = getpwuid(os.stat(socket).st_uid).pw_name - self.echo( - "Using socket {}, owned by user {}".format( - socket, socket_owner) - ) else: host = host or 'localhost' port = port or 3306 @@ -1009,6 +996,9 @@ def get_last_query(self): @click.option('--ssh-port', default=22, help='Port to connect to ssh server.') @click.option('--ssh-password', help='Password to connect to ssh server.') @click.option('--ssh-key-filename', help='Private key filename (identify file) for the ssh connection.') +@click.option('--ssh-config-path', help='Path to ssh configuration.', + default=os.path.expanduser('~') + '/.ssh/config') +@click.option('--ssh-config-host', help='Host to connect to ssh server reading from ssh configuration.') @click.option('--ssl-ca', help='CA file in PEM format.', type=click.Path(exists=True)) @click.option('--ssl-capath', help='CA directory.') @@ -1030,6 +1020,8 @@ def get_last_query(self): help='Use DSN configured into the [alias_dsn] section of myclirc file.') @click.option('--list-dsn', 'list_dsn', is_flag=True, help='list of DSN configured into the [alias_dsn] section of myclirc file.') +@click.option('--list-ssh-config', 'list_ssh_config', is_flag=True, + help='list ssh configurations in the ssh config (requires paramiko).') @click.option('-R', '--prompt', 'prompt', help='Prompt format (Default: "{0}").'.format( MyCli.default_prompt)) @@ -1062,7 +1054,7 @@ def cli(database, user, host, port, socket, password, dbname, ssl_ca, ssl_capath, ssl_cert, ssl_key, ssl_cipher, ssl_verify_server_cert, table, csv, warn, execute, myclirc, dsn, list_dsn, ssh_user, ssh_host, ssh_port, ssh_password, - ssh_key_filename): + ssh_key_filename, list_ssh_config, ssh_config_path, ssh_config_host): """A MySQL terminal client with auto-completion and syntax highlighting. \b @@ -1099,6 +1091,31 @@ def cli(database, user, host, port, socket, password, dbname, else: click.secho(alias) sys.exit(0) + if list_ssh_config: + if not paramiko: + click.secho( + "This features requires paramiko. Please install paramiko and try again.", + err=True, fg='red' + ) + exit(1) + try: + ssh_config = paramiko.config.SSHConfig().from_path(ssh_config_path) + except paramiko.ssh_exception.ConfigParseError as err: + click.secho('Invalid SSH configuration file. ' + 'Please check the SSH configuration file.', + err=True, fg='red') + exit(1) + except FileNotFoundError as e: + click.secho(str(e), err=True, fg='red') + exit(1) + for host in ssh_config.get_hostnames(): + if verbose: + host_config = ssh_config.lookup(host) + click.secho("{} : {}".format( + host, host_config.get('hostname'))) + else: + click.secho(host) + sys.exit(0) # Choose which ever one has a valid value. database = dbname or database @@ -1149,6 +1166,32 @@ def cli(database, user, host, port, socket, password, dbname, if not port: port = uri.port + if ssh_config_host: + if not paramiko: + click.secho( + "This features requires paramiko. Please install paramiko and try again.", + err=True, fg='red' + ) + exit(1) + try: + ssh_config = paramiko.config.SSHConfig().from_path(ssh_config_path) + except paramiko.ssh_exception.ConfigParseError as err: + click.secho('Invalid SSH configuration file. ' + 'Please check the SSH configuration file.', + err=True, fg='red') + exit(1) + except FileNotFoundError as e: + click.secho(str(e), err=True, fg='red') + exit(1) + ssh_config = ssh_config.lookup(ssh_config_host) + ssh_host = ssh_host if ssh_host else ssh_config.get('hostname') + ssh_user = ssh_user if ssh_user else ssh_config.get('user') + if ssh_config.get('port') and ssh_port == 22: + # port has a default value, overwrite it if it's in the config + ssh_port = int(ssh_config.get('port')) + ssh_key_filename = ssh_key_filename if ssh_key_filename else ssh_config.get( + 'identityfile', [''])[0] + if not paramiko and ssh_host: click.secho( "Cannot use SSH transport because paramiko isn't installed, " diff --git a/mycli/packages/filepaths.py b/mycli/packages/filepaths.py index 26b77c40..79fe26dc 100644 --- a/mycli/packages/filepaths.py +++ b/mycli/packages/filepaths.py @@ -103,4 +103,4 @@ def guess_socket_location(): if name.startswith("mysql") and ext in ('.socket', '.sock'): return os.path.join(r, filename) dirs[:] = [d for d in dirs if d.startswith("mysql")] - raise FileNotFoundError + return None diff --git a/mycli/sqlexecute.py b/mycli/sqlexecute.py index 88db9dc4..f38da6f6 100644 --- a/mycli/sqlexecute.py +++ b/mycli/sqlexecute.py @@ -191,7 +191,6 @@ def run(self, statement): if not cur.nextset() or (not cur.rowcount and cur.description is None): break - def get_result(self, cursor): """Get the current result's data from the cursor.""" title = headers = None From a3cf82a4f34c8f86ea6ebc524e9aac4c08e340a1 Mon Sep 17 00:00:00 2001 From: Georgy Frolov Date: Sun, 10 May 2020 22:32:31 +0300 Subject: [PATCH 7/8] read myclird section in .cnf files --- mycli/config.py | 47 ++++++++++++++++++++++++++++++++--------------- mycli/main.py | 2 +- 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/mycli/config.py b/mycli/config.py index 03fb502a..e0f2d1fc 100644 --- a/mycli/config.py +++ b/mycli/config.py @@ -1,3 +1,4 @@ +import io import shutil from copy import copy from io import BytesIO, TextIOWrapper @@ -6,6 +7,7 @@ from os.path import exists import struct import sys +from typing import Union from configobj import ConfigObj, ConfigObjError from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes @@ -59,23 +61,34 @@ def read_config_file(f, list_values=True): return config -def get_included_configs(config_path) -> list: +def get_included_configs(config_file: Union[str, io.TextIOWrapper]) -> list: """Get a list of configuration files that are included into config_path - with !includedir directive.""" - if not os.path.exists(config_path): + with !includedir directive. + + "Normal" configs should be passed as file paths. The only exception + is .mylogin which is decoded into a stream. However, it never + contains include directives and so will be ignored by this + function. + + """ + if not isinstance(config_file, str) or not os.path.isfile(config_file): return [] included_configs = [] - with open(config_path) as f: - include_directives = filter( - lambda s: s.startswith('!includedir'), - f - ) - dirs = map(lambda s: s.strip().split()[-1], include_directives) - dirs = filter(os.path.isdir, dirs) - for dir in dirs: - for filename in os.listdir(dir): - if filename.endswith('.cnf'): - included_configs.append(os.path.join(dir, filename)) + + try: + with open(config_file) as f: + include_directives = filter( + lambda s: s.startswith('!includedir'), + f + ) + dirs = map(lambda s: s.strip().split()[-1], include_directives) + dirs = filter(os.path.isdir, dirs) + for dir in dirs: + for filename in os.listdir(dir): + if filename.endswith('.cnf'): + included_configs.append(os.path.join(dir, filename)) + except (PermissionError, UnicodeDecodeError): + pass return included_configs @@ -86,8 +99,12 @@ def read_config_files(files, list_values=True): _files = copy(files) while _files: _file = _files.pop(0) - _files = get_included_configs(_file) + _files _config = read_config_file(_file, list_values=list_values) + + # expand includes only if we were able to parse config + # (otherwise we'll just encounter the same errors again) + if config is not None: + _files = get_included_configs(_file) + _files if bool(_config) is True: config.merge(_config) config.filename = _config.filename diff --git a/mycli/main.py b/mycli/main.py index 55afedd5..1fe2a848 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -318,7 +318,7 @@ def read_my_cnf_files(self, files, keys): """ cnf = read_config_files(files, list_values=False) - sections = ['client'] + sections = ['client', 'mysqld'] if self.login_path and self.login_path != 'client': sections.append(self.login_path) From 977e409aa8abd7804d1ee62809e791e23f2716db Mon Sep 17 00:00:00 2001 From: Georgy Frolov Date: Sun, 24 May 2020 00:00:24 +0300 Subject: [PATCH 8/8] Don't try to connect to socket over SSH --- mycli/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mycli/main.py b/mycli/main.py index 1fe2a848..00cffd0a 100755 --- a/mycli/main.py +++ b/mycli/main.py @@ -383,7 +383,8 @@ def connect(self, database='', user='', passwd='', host='', port='', # Fall back to config values only if user did not specify a value. database = database or cnf['database'] - if port or host: + # Socket interface not supported for SSH connections + if port or host or ssh_host or ssh_port: socket = '' else: socket = socket or cnf['socket'] or guess_socket_location()