# Add some packages to LPy64 (Jupyter)

dev-linux
Mikhail 2 years ago
parent 5f45962173
commit 0152bbd027

@ -0,0 +1,8 @@
#!/media/nd/Acer/pyOpenRPA/dev-linux/Resources/LPy64-3105/bin/python3.10
# -*- coding: utf-8 -*-
import re
import sys
from IPython import start_ipython
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(start_ipython())

@ -0,0 +1,8 @@
#!/media/nd/Acer/pyOpenRPA/dev-linux/Resources/LPy64-3105/bin/python3.10
# -*- coding: utf-8 -*-
import re
import sys
from IPython import start_ipython
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(start_ipython())

@ -0,0 +1,8 @@
#!/media/nd/Acer/pyOpenRPA/dev-linux/Resources/LPy64-3105/bin/python3.10
# -*- coding: utf-8 -*-
import re
import sys
from jsonschema.cli import main
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(main())

@ -0,0 +1,8 @@
#!/media/nd/Acer/pyOpenRPA/dev-linux/Resources/LPy64-3105/bin/python3.10
# -*- coding: utf-8 -*-
import re
import sys
from jupyter_core.command import main
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(main())

@ -0,0 +1,8 @@
#!/media/nd/Acer/pyOpenRPA/dev-linux/Resources/LPy64-3105/bin/python3.10
# -*- coding: utf-8 -*-
import re
import sys
from notebook.bundler.bundlerextensions import main
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(main())

@ -0,0 +1,8 @@
#!/media/nd/Acer/pyOpenRPA/dev-linux/Resources/LPy64-3105/bin/python3.10
# -*- coding: utf-8 -*-
import re
import sys
from nbconvert.nbconvertapp import dejavu_main
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(dejavu_main())

@ -0,0 +1,8 @@
#!/media/nd/Acer/pyOpenRPA/dev-linux/Resources/LPy64-3105/bin/python3.10
# -*- coding: utf-8 -*-
import re
import sys
from nbclient.cli import main
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(main())

@ -0,0 +1,8 @@
#!/media/nd/Acer/pyOpenRPA/dev-linux/Resources/LPy64-3105/bin/python3.10
# -*- coding: utf-8 -*-
import re
import sys
from jupyter_client.kernelapp import main
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(main())

@ -0,0 +1,8 @@
#!/media/nd/Acer/pyOpenRPA/dev-linux/Resources/LPy64-3105/bin/python3.10
# -*- coding: utf-8 -*-
import re
import sys
from jupyter_client.kernelspecapp import KernelSpecApp
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(KernelSpecApp.launch_instance())

@ -0,0 +1,8 @@
#!/media/nd/Acer/pyOpenRPA/dev-linux/Resources/LPy64-3105/bin/python3.10
# -*- coding: utf-8 -*-
import re
import sys
from jupyter_core.migrate import main
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(main())

@ -0,0 +1,8 @@
#!/media/nd/Acer/pyOpenRPA/dev-linux/Resources/LPy64-3105/bin/python3.10
# -*- coding: utf-8 -*-
import re
import sys
from nbconvert.nbconvertapp import main
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(main())

@ -0,0 +1,8 @@
#!/media/nd/Acer/pyOpenRPA/dev-linux/Resources/LPy64-3105/bin/python3.10
# -*- coding: utf-8 -*-
import re
import sys
from notebook.nbextensions import main
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(main())

@ -0,0 +1,8 @@
#!/media/nd/Acer/pyOpenRPA/dev-linux/Resources/LPy64-3105/bin/python3.10
# -*- coding: utf-8 -*-
import re
import sys
from notebook.notebookapp import main
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(main())

@ -0,0 +1,8 @@
#!/media/nd/Acer/pyOpenRPA/dev-linux/Resources/LPy64-3105/bin/python3.10
# -*- coding: utf-8 -*-
import re
import sys
from jupyter_client.runapp import RunApp
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(RunApp.launch_instance())

@ -0,0 +1,8 @@
#!/media/nd/Acer/pyOpenRPA/dev-linux/Resources/LPy64-3105/bin/python3.10
# -*- coding: utf-8 -*-
import re
import sys
from notebook.serverextensions import main
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(main())

@ -0,0 +1,8 @@
#!/media/nd/Acer/pyOpenRPA/dev-linux/Resources/LPy64-3105/bin/python3.10
# -*- coding: utf-8 -*-
import re
import sys
from jupyter_core.troubleshoot import main
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(main())

@ -0,0 +1,8 @@
#!/media/nd/Acer/pyOpenRPA/dev-linux/Resources/LPy64-3105/bin/python3.10
# -*- coding: utf-8 -*-
import re
import sys
from nbformat.sign import TrustNotebookApp
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(TrustNotebookApp.launch_instance())

@ -0,0 +1,8 @@
#!/media/nd/Acer/pyOpenRPA/dev-linux/Resources/LPy64-3105/bin/python3.10
# -*- coding: utf-8 -*-
import re
import sys
from pygments.cmdline import main
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(main())

@ -0,0 +1,8 @@
#!/media/nd/Acer/pyOpenRPA/dev-linux/Resources/LPy64-3105/bin/python3.10
# -*- coding: utf-8 -*-
import re
import sys
from send2trash.__main__ import main
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(main())

@ -0,0 +1,156 @@
"""
IPython: tools for interactive and parallel computing in Python.
https://ipython.org
"""
#-----------------------------------------------------------------------------
# Copyright (c) 2008-2011, IPython Development Team.
# Copyright (c) 2001-2007, Fernando Perez <fernando.perez@colorado.edu>
# Copyright (c) 2001, Janko Hauser <jhauser@zscout.de>
# Copyright (c) 2001, Nathaniel Gray <n8gray@caltech.edu>
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
import os
import sys
#-----------------------------------------------------------------------------
# Setup everything
#-----------------------------------------------------------------------------
# Don't forget to also update setup.py when this changes!
if sys.version_info < (3, 8):
raise ImportError(
"""
IPython 8+ supports Python 3.8 and above, following NEP 29.
When using Python 2.7, please install IPython 5.x LTS Long Term Support version.
Python 3.3 and 3.4 were supported up to IPython 6.x.
Python 3.5 was supported with IPython 7.0 to 7.9.
Python 3.6 was supported with IPython up to 7.16.
Python 3.7 was still supported with the 7.x branch.
See IPython `README.rst` file for more information:
https://github.com/ipython/ipython/blob/master/README.rst
"""
)
#-----------------------------------------------------------------------------
# Setup the top level names
#-----------------------------------------------------------------------------
from .core.getipython import get_ipython
from .core import release
from .core.application import Application
from .terminal.embed import embed
from .core.interactiveshell import InteractiveShell
from .utils.sysinfo import sys_info
from .utils.frame import extract_module_locals
# Release data
__author__ = '%s <%s>' % (release.author, release.author_email)
__license__ = release.license
__version__ = release.version
version_info = release.version_info
# list of CVEs that should have been patched in this release.
# this is informational and should not be relied upon.
__patched_cves__ = {"CVE-2022-21699"}
def embed_kernel(module=None, local_ns=None, **kwargs):
"""Embed and start an IPython kernel in a given scope.
If you don't want the kernel to initialize the namespace
from the scope of the surrounding function,
and/or you want to load full IPython configuration,
you probably want `IPython.start_kernel()` instead.
Parameters
----------
module : types.ModuleType, optional
The module to load into IPython globals (default: caller)
local_ns : dict, optional
The namespace to load into IPython user namespace (default: caller)
**kwargs : various, optional
Further keyword args are relayed to the IPKernelApp constructor,
allowing configuration of the Kernel. Will only have an effect
on the first embed_kernel call for a given process.
"""
(caller_module, caller_locals) = extract_module_locals(1)
if module is None:
module = caller_module
if local_ns is None:
local_ns = caller_locals
# Only import .zmq when we really need it
from ipykernel.embed import embed_kernel as real_embed_kernel
real_embed_kernel(module=module, local_ns=local_ns, **kwargs)
def start_ipython(argv=None, **kwargs):
"""Launch a normal IPython instance (as opposed to embedded)
`IPython.embed()` puts a shell in a particular calling scope,
such as a function or method for debugging purposes,
which is often not desirable.
`start_ipython()` does full, regular IPython initialization,
including loading startup files, configuration, etc.
much of which is skipped by `embed()`.
This is a public API method, and will survive implementation changes.
Parameters
----------
argv : list or None, optional
If unspecified or None, IPython will parse command-line options from sys.argv.
To prevent any command-line parsing, pass an empty list: `argv=[]`.
user_ns : dict, optional
specify this dictionary to initialize the IPython user namespace with particular values.
**kwargs : various, optional
Any other kwargs will be passed to the Application constructor,
such as `config`.
"""
from IPython.terminal.ipapp import launch_new_instance
return launch_new_instance(argv=argv, **kwargs)
def start_kernel(argv=None, **kwargs):
"""Launch a normal IPython kernel instance (as opposed to embedded)
`IPython.embed_kernel()` puts a shell in a particular calling scope,
such as a function or method for debugging purposes,
which is often not desirable.
`start_kernel()` does full, regular IPython initialization,
including loading startup files, configuration, etc.
much of which is skipped by `embed()`.
Parameters
----------
argv : list or None, optional
If unspecified or None, IPython will parse command-line options from sys.argv.
To prevent any command-line parsing, pass an empty list: `argv=[]`.
user_ns : dict, optional
specify this dictionary to initialize the IPython user namespace with particular values.
**kwargs : various, optional
Any other kwargs will be passed to the Application constructor,
such as `config`.
"""
import warnings
warnings.warn(
"start_kernel is deprecated since IPython 8.0, use from `ipykernel.kernelapp.launch_new_instance`",
DeprecationWarning,
stacklevel=2,
)
from ipykernel.kernelapp import launch_new_instance
return launch_new_instance(argv=argv, **kwargs)

@ -0,0 +1,14 @@
# encoding: utf-8
"""Terminal-based IPython entry point.
"""
#-----------------------------------------------------------------------------
# Copyright (c) 2012, IPython Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
from IPython import start_ipython
start_ipython()

@ -0,0 +1,87 @@
import builtins
import inspect
import os
import pathlib
import shutil
import sys
import types
import pytest
# Must register before it gets imported
pytest.register_assert_rewrite("IPython.testing.tools")
from .testing import tools
def pytest_collection_modifyitems(items):
"""This function is automatically run by pytest passing all collected test
functions.
We use it to add asyncio marker to all async tests and assert we don't use
test functions that are async generators which wouldn't make sense.
"""
for item in items:
if inspect.iscoroutinefunction(item.obj):
item.add_marker("asyncio")
assert not inspect.isasyncgenfunction(item.obj)
def get_ipython():
from .terminal.interactiveshell import TerminalInteractiveShell
if TerminalInteractiveShell._instance:
return TerminalInteractiveShell.instance()
config = tools.default_config()
config.TerminalInteractiveShell.simple_prompt = True
# Create and initialize our test-friendly IPython instance.
shell = TerminalInteractiveShell.instance(config=config)
return shell
@pytest.fixture(scope='session', autouse=True)
def work_path():
path = pathlib.Path("./tmp-ipython-pytest-profiledir")
os.environ["IPYTHONDIR"] = str(path.absolute())
if path.exists():
raise ValueError('IPython dir temporary path already exists ! Did previous test run exit successfully ?')
path.mkdir()
yield
shutil.rmtree(str(path.resolve()))
def nopage(strng, start=0, screen_lines=0, pager_cmd=None):
if isinstance(strng, dict):
strng = strng.get("text/plain", "")
print(strng)
def xsys(self, cmd):
"""Replace the default system call with a capturing one for doctest.
"""
# We use getoutput, but we need to strip it because pexpect captures
# the trailing newline differently from commands.getoutput
print(self.getoutput(cmd, split=False, depth=1).rstrip(), end="", file=sys.stdout)
sys.stdout.flush()
# for things to work correctly we would need this as a session fixture;
# unfortunately this will fail on some test that get executed as _collection_
# time (before the fixture run), in particular parametrized test that contain
# yields. so for now execute at import time.
#@pytest.fixture(autouse=True, scope='session')
def inject():
builtins.get_ipython = get_ipython
builtins._ip = get_ipython()
builtins.ip = get_ipython()
builtins.ip.system = types.MethodType(xsys, ip)
builtins.ip.builtin_trap.activate()
from .core import page
page.pager_page = nopage
# yield
inject()

@ -0,0 +1,12 @@
"""
Shim to maintain backwards compatibility with old IPython.consoleapp imports.
"""
# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.
from warnings import warn
warn("The `IPython.consoleapp` package has been deprecated since IPython 4.0."
"You should import from jupyter_client.consoleapp instead.", stacklevel=2)
from jupyter_client.consoleapp import *

@ -0,0 +1,258 @@
# encoding: utf-8
"""
System command aliases.
Authors:
* Fernando Perez
* Brian Granger
"""
#-----------------------------------------------------------------------------
# Copyright (C) 2008-2011 The IPython Development Team
#
# Distributed under the terms of the BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
import os
import re
import sys
from traitlets.config.configurable import Configurable
from .error import UsageError
from traitlets import List, Instance
from logging import error
#-----------------------------------------------------------------------------
# Utilities
#-----------------------------------------------------------------------------
# This is used as the pattern for calls to split_user_input.
shell_line_split = re.compile(r'^(\s*)()(\S+)(.*$)')
def default_aliases():
"""Return list of shell aliases to auto-define.
"""
# Note: the aliases defined here should be safe to use on a kernel
# regardless of what frontend it is attached to. Frontends that use a
# kernel in-process can define additional aliases that will only work in
# their case. For example, things like 'less' or 'clear' that manipulate
# the terminal should NOT be declared here, as they will only work if the
# kernel is running inside a true terminal, and not over the network.
if os.name == 'posix':
default_aliases = [('mkdir', 'mkdir'), ('rmdir', 'rmdir'),
('mv', 'mv'), ('rm', 'rm'), ('cp', 'cp'),
('cat', 'cat'),
]
# Useful set of ls aliases. The GNU and BSD options are a little
# different, so we make aliases that provide as similar as possible
# behavior in ipython, by passing the right flags for each platform
if sys.platform.startswith('linux'):
ls_aliases = [('ls', 'ls -F --color'),
# long ls
('ll', 'ls -F -o --color'),
# ls normal files only
('lf', 'ls -F -o --color %l | grep ^-'),
# ls symbolic links
('lk', 'ls -F -o --color %l | grep ^l'),
# directories or links to directories,
('ldir', 'ls -F -o --color %l | grep /$'),
# things which are executable
('lx', 'ls -F -o --color %l | grep ^-..x'),
]
elif sys.platform.startswith('openbsd') or sys.platform.startswith('netbsd'):
# OpenBSD, NetBSD. The ls implementation on these platforms do not support
# the -G switch and lack the ability to use colorized output.
ls_aliases = [('ls', 'ls -F'),
# long ls
('ll', 'ls -F -l'),
# ls normal files only
('lf', 'ls -F -l %l | grep ^-'),
# ls symbolic links
('lk', 'ls -F -l %l | grep ^l'),
# directories or links to directories,
('ldir', 'ls -F -l %l | grep /$'),
# things which are executable
('lx', 'ls -F -l %l | grep ^-..x'),
]
else:
# BSD, OSX, etc.
ls_aliases = [('ls', 'ls -F -G'),
# long ls
('ll', 'ls -F -l -G'),
# ls normal files only
('lf', 'ls -F -l -G %l | grep ^-'),
# ls symbolic links
('lk', 'ls -F -l -G %l | grep ^l'),
# directories or links to directories,
('ldir', 'ls -F -G -l %l | grep /$'),
# things which are executable
('lx', 'ls -F -l -G %l | grep ^-..x'),
]
default_aliases = default_aliases + ls_aliases
elif os.name in ['nt', 'dos']:
default_aliases = [('ls', 'dir /on'),
('ddir', 'dir /ad /on'), ('ldir', 'dir /ad /on'),
('mkdir', 'mkdir'), ('rmdir', 'rmdir'),
('echo', 'echo'), ('ren', 'ren'), ('copy', 'copy'),
]
else:
default_aliases = []
return default_aliases
class AliasError(Exception):
pass
class InvalidAliasError(AliasError):
pass
class Alias(object):
"""Callable object storing the details of one alias.
Instances are registered as magic functions to allow use of aliases.
"""
# Prepare blacklist
blacklist = {'cd','popd','pushd','dhist','alias','unalias'}
def __init__(self, shell, name, cmd):
self.shell = shell
self.name = name
self.cmd = cmd
self.__doc__ = "Alias for `!{}`".format(cmd)
self.nargs = self.validate()
def validate(self):
"""Validate the alias, and return the number of arguments."""
if self.name in self.blacklist:
raise InvalidAliasError("The name %s can't be aliased "
"because it is a keyword or builtin." % self.name)
try:
caller = self.shell.magics_manager.magics['line'][self.name]
except KeyError:
pass
else:
if not isinstance(caller, Alias):
raise InvalidAliasError("The name %s can't be aliased "
"because it is another magic command." % self.name)
if not (isinstance(self.cmd, str)):
raise InvalidAliasError("An alias command must be a string, "
"got: %r" % self.cmd)
nargs = self.cmd.count('%s') - self.cmd.count('%%s')
if (nargs > 0) and (self.cmd.find('%l') >= 0):
raise InvalidAliasError('The %s and %l specifiers are mutually '
'exclusive in alias definitions.')
return nargs
def __repr__(self):
return "<alias {} for {!r}>".format(self.name, self.cmd)
def __call__(self, rest=''):
cmd = self.cmd
nargs = self.nargs
# Expand the %l special to be the user's input line
if cmd.find('%l') >= 0:
cmd = cmd.replace('%l', rest)
rest = ''
if nargs==0:
if cmd.find('%%s') >= 1:
cmd = cmd.replace('%%s', '%s')
# Simple, argument-less aliases
cmd = '%s %s' % (cmd, rest)
else:
# Handle aliases with positional arguments
args = rest.split(None, nargs)
if len(args) < nargs:
raise UsageError('Alias <%s> requires %s arguments, %s given.' %
(self.name, nargs, len(args)))
cmd = '%s %s' % (cmd % tuple(args[:nargs]),' '.join(args[nargs:]))
self.shell.system(cmd)
#-----------------------------------------------------------------------------
# Main AliasManager class
#-----------------------------------------------------------------------------
class AliasManager(Configurable):
default_aliases = List(default_aliases()).tag(config=True)
user_aliases = List(default_value=[]).tag(config=True)
shell = Instance('IPython.core.interactiveshell.InteractiveShellABC', allow_none=True)
def __init__(self, shell=None, **kwargs):
super(AliasManager, self).__init__(shell=shell, **kwargs)
# For convenient access
self.linemagics = self.shell.magics_manager.magics['line']
self.init_aliases()
def init_aliases(self):
# Load default & user aliases
for name, cmd in self.default_aliases + self.user_aliases:
if cmd.startswith('ls ') and self.shell.colors == 'NoColor':
cmd = cmd.replace(' --color', '')
self.soft_define_alias(name, cmd)
@property
def aliases(self):
return [(n, func.cmd) for (n, func) in self.linemagics.items()
if isinstance(func, Alias)]
def soft_define_alias(self, name, cmd):
"""Define an alias, but don't raise on an AliasError."""
try:
self.define_alias(name, cmd)
except AliasError as e:
error("Invalid alias: %s" % e)
def define_alias(self, name, cmd):
"""Define a new alias after validating it.
This will raise an :exc:`AliasError` if there are validation
problems.
"""
caller = Alias(shell=self.shell, name=name, cmd=cmd)
self.shell.magics_manager.register_function(caller, magic_kind='line',
magic_name=name)
def get_alias(self, name):
"""Return an alias, or None if no alias by that name exists."""
aname = self.linemagics.get(name, None)
return aname if isinstance(aname, Alias) else None
def is_alias(self, name):
"""Return whether or not a given name has been defined as an alias"""
return self.get_alias(name) is not None
def undefine_alias(self, name):
if self.is_alias(name):
del self.linemagics[name]
else:
raise ValueError('%s is not an alias' % name)
def clear_aliases(self):
for name, cmd in self.aliases:
self.undefine_alias(name)
def retrieve_alias(self, name):
"""Retrieve the command to which an alias expands."""
caller = self.get_alias(name)
if caller:
return caller.cmd
else:
raise ValueError('%s is not an alias' % name)

@ -0,0 +1,490 @@
# encoding: utf-8
"""
An application for IPython.
All top-level applications should use the classes in this module for
handling configuration and creating configurables.
The job of an :class:`Application` is to create the master configuration
object and then create the configurable objects, passing the config to them.
"""
# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.
import atexit
from copy import deepcopy
import glob
import logging
import os
import shutil
import sys
from pathlib import Path
from traitlets.config.application import Application, catch_config_error
from traitlets.config.loader import ConfigFileNotFound, PyFileConfigLoader
from IPython.core import release, crashhandler
from IPython.core.profiledir import ProfileDir, ProfileDirError
from IPython.paths import get_ipython_dir, get_ipython_package_dir
from IPython.utils.path import ensure_dir_exists
from traitlets import (
List, Unicode, Type, Bool, Set, Instance, Undefined,
default, observe,
)
if os.name == "nt":
programdata = os.environ.get("PROGRAMDATA", None)
if programdata is not None:
SYSTEM_CONFIG_DIRS = [str(Path(programdata) / "ipython")]
else: # PROGRAMDATA is not defined by default on XP.
SYSTEM_CONFIG_DIRS = []
else:
SYSTEM_CONFIG_DIRS = [
"/usr/local/etc/ipython",
"/etc/ipython",
]
ENV_CONFIG_DIRS = []
_env_config_dir = os.path.join(sys.prefix, 'etc', 'ipython')
if _env_config_dir not in SYSTEM_CONFIG_DIRS:
# only add ENV_CONFIG if sys.prefix is not already included
ENV_CONFIG_DIRS.append(_env_config_dir)
_envvar = os.environ.get('IPYTHON_SUPPRESS_CONFIG_ERRORS')
if _envvar in {None, ''}:
IPYTHON_SUPPRESS_CONFIG_ERRORS = None
else:
if _envvar.lower() in {'1','true'}:
IPYTHON_SUPPRESS_CONFIG_ERRORS = True
elif _envvar.lower() in {'0','false'} :
IPYTHON_SUPPRESS_CONFIG_ERRORS = False
else:
sys.exit("Unsupported value for environment variable: 'IPYTHON_SUPPRESS_CONFIG_ERRORS' is set to '%s' which is none of {'0', '1', 'false', 'true', ''}."% _envvar )
# aliases and flags
base_aliases = {}
if isinstance(Application.aliases, dict):
# traitlets 5
base_aliases.update(Application.aliases)
base_aliases.update(
{
"profile-dir": "ProfileDir.location",
"profile": "BaseIPythonApplication.profile",
"ipython-dir": "BaseIPythonApplication.ipython_dir",
"log-level": "Application.log_level",
"config": "BaseIPythonApplication.extra_config_file",
}
)
base_flags = dict()
if isinstance(Application.flags, dict):
# traitlets 5
base_flags.update(Application.flags)
base_flags.update(
dict(
debug=(
{"Application": {"log_level": logging.DEBUG}},
"set log level to logging.DEBUG (maximize logging output)",
),
quiet=(
{"Application": {"log_level": logging.CRITICAL}},
"set log level to logging.CRITICAL (minimize logging output)",
),
init=(
{
"BaseIPythonApplication": {
"copy_config_files": True,
"auto_create": True,
}
},
"""Initialize profile with default config files. This is equivalent
to running `ipython profile create <profile>` prior to startup.
""",
),
)
)
class ProfileAwareConfigLoader(PyFileConfigLoader):
"""A Python file config loader that is aware of IPython profiles."""
def load_subconfig(self, fname, path=None, profile=None):
if profile is not None:
try:
profile_dir = ProfileDir.find_profile_dir_by_name(
get_ipython_dir(),
profile,
)
except ProfileDirError:
return
path = profile_dir.location
return super(ProfileAwareConfigLoader, self).load_subconfig(fname, path=path)
class BaseIPythonApplication(Application):
name = u'ipython'
description = Unicode(u'IPython: an enhanced interactive Python shell.')
version = Unicode(release.version)
aliases = base_aliases
flags = base_flags
classes = List([ProfileDir])
# enable `load_subconfig('cfg.py', profile='name')`
python_config_loader_class = ProfileAwareConfigLoader
# Track whether the config_file has changed,
# because some logic happens only if we aren't using the default.
config_file_specified = Set()
config_file_name = Unicode()
@default('config_file_name')
def _config_file_name_default(self):
return self.name.replace('-','_') + u'_config.py'
@observe('config_file_name')
def _config_file_name_changed(self, change):
if change['new'] != change['old']:
self.config_file_specified.add(change['new'])
# The directory that contains IPython's builtin profiles.
builtin_profile_dir = Unicode(
os.path.join(get_ipython_package_dir(), u'config', u'profile', u'default')
)
config_file_paths = List(Unicode())
@default('config_file_paths')
def _config_file_paths_default(self):
return []
extra_config_file = Unicode(
help="""Path to an extra config file to load.
If specified, load this config file in addition to any other IPython config.
""").tag(config=True)
@observe('extra_config_file')
def _extra_config_file_changed(self, change):
old = change['old']
new = change['new']
try:
self.config_files.remove(old)
except ValueError:
pass
self.config_file_specified.add(new)
self.config_files.append(new)
profile = Unicode(u'default',
help="""The IPython profile to use."""
).tag(config=True)
@observe('profile')
def _profile_changed(self, change):
self.builtin_profile_dir = os.path.join(
get_ipython_package_dir(), u'config', u'profile', change['new']
)
add_ipython_dir_to_sys_path = Bool(
False,
"""Should the IPython profile directory be added to sys path ?
This option was non-existing before IPython 8.0, and ipython_dir was added to
sys path to allow import of extensions present there. This was historical
baggage from when pip did not exist. This now default to false,
but can be set to true for legacy reasons.
""",
).tag(config=True)
ipython_dir = Unicode(
help="""
The name of the IPython directory. This directory is used for logging
configuration (through profiles), history storage, etc. The default
is usually $HOME/.ipython. This option can also be specified through
the environment variable IPYTHONDIR.
"""
).tag(config=True)
@default('ipython_dir')
def _ipython_dir_default(self):
d = get_ipython_dir()
self._ipython_dir_changed({
'name': 'ipython_dir',
'old': d,
'new': d,
})
return d
_in_init_profile_dir = False
profile_dir = Instance(ProfileDir, allow_none=True)
@default('profile_dir')
def _profile_dir_default(self):
# avoid recursion
if self._in_init_profile_dir:
return
# profile_dir requested early, force initialization
self.init_profile_dir()
return self.profile_dir
overwrite = Bool(False,
help="""Whether to overwrite existing config files when copying"""
).tag(config=True)
auto_create = Bool(False,
help="""Whether to create profile dir if it doesn't exist"""
).tag(config=True)
config_files = List(Unicode())
@default('config_files')
def _config_files_default(self):
return [self.config_file_name]
copy_config_files = Bool(False,
help="""Whether to install the default config files into the profile dir.
If a new profile is being created, and IPython contains config files for that
profile, then they will be staged into the new directory. Otherwise,
default config files will be automatically generated.
""").tag(config=True)
verbose_crash = Bool(False,
help="""Create a massive crash report when IPython encounters what may be an
internal error. The default is to append a short message to the
usual traceback""").tag(config=True)
# The class to use as the crash handler.
crash_handler_class = Type(crashhandler.CrashHandler)
@catch_config_error
def __init__(self, **kwargs):
super(BaseIPythonApplication, self).__init__(**kwargs)
# ensure current working directory exists
try:
os.getcwd()
except:
# exit if cwd doesn't exist
self.log.error("Current working directory doesn't exist.")
self.exit(1)
#-------------------------------------------------------------------------
# Various stages of Application creation
#-------------------------------------------------------------------------
def init_crash_handler(self):
"""Create a crash handler, typically setting sys.excepthook to it."""
self.crash_handler = self.crash_handler_class(self)
sys.excepthook = self.excepthook
def unset_crashhandler():
sys.excepthook = sys.__excepthook__
atexit.register(unset_crashhandler)
def excepthook(self, etype, evalue, tb):
"""this is sys.excepthook after init_crashhandler
set self.verbose_crash=True to use our full crashhandler, instead of
a regular traceback with a short message (crash_handler_lite)
"""
if self.verbose_crash:
return self.crash_handler(etype, evalue, tb)
else:
return crashhandler.crash_handler_lite(etype, evalue, tb)
@observe('ipython_dir')
def _ipython_dir_changed(self, change):
old = change['old']
new = change['new']
if old is not Undefined:
str_old = os.path.abspath(old)
if str_old in sys.path:
sys.path.remove(str_old)
if self.add_ipython_dir_to_sys_path:
str_path = os.path.abspath(new)
sys.path.append(str_path)
ensure_dir_exists(new)
readme = os.path.join(new, "README")
readme_src = os.path.join(
get_ipython_package_dir(), "config", "profile", "README"
)
if not os.path.exists(readme) and os.path.exists(readme_src):
shutil.copy(readme_src, readme)
for d in ("extensions", "nbextensions"):
path = os.path.join(new, d)
try:
ensure_dir_exists(path)
except OSError as e:
# this will not be EEXIST
self.log.error("couldn't create path %s: %s", path, e)
self.log.debug("IPYTHONDIR set to: %s" % new)
def load_config_file(self, suppress_errors=IPYTHON_SUPPRESS_CONFIG_ERRORS):
"""Load the config file.
By default, errors in loading config are handled, and a warning
printed on screen. For testing, the suppress_errors option is set
to False, so errors will make tests fail.
`suppress_errors` default value is to be `None` in which case the
behavior default to the one of `traitlets.Application`.
The default value can be set :
- to `False` by setting 'IPYTHON_SUPPRESS_CONFIG_ERRORS' environment variable to '0', or 'false' (case insensitive).
- to `True` by setting 'IPYTHON_SUPPRESS_CONFIG_ERRORS' environment variable to '1' or 'true' (case insensitive).
- to `None` by setting 'IPYTHON_SUPPRESS_CONFIG_ERRORS' environment variable to '' (empty string) or leaving it unset.
Any other value are invalid, and will make IPython exit with a non-zero return code.
"""
self.log.debug("Searching path %s for config files", self.config_file_paths)
base_config = 'ipython_config.py'
self.log.debug("Attempting to load config file: %s" %
base_config)
try:
if suppress_errors is not None:
old_value = Application.raise_config_file_errors
Application.raise_config_file_errors = not suppress_errors;
Application.load_config_file(
self,
base_config,
path=self.config_file_paths
)
except ConfigFileNotFound:
# ignore errors loading parent
self.log.debug("Config file %s not found", base_config)
pass
if suppress_errors is not None:
Application.raise_config_file_errors = old_value
for config_file_name in self.config_files:
if not config_file_name or config_file_name == base_config:
continue
self.log.debug("Attempting to load config file: %s" %
self.config_file_name)
try:
Application.load_config_file(
self,
config_file_name,
path=self.config_file_paths
)
except ConfigFileNotFound:
# Only warn if the default config file was NOT being used.
if config_file_name in self.config_file_specified:
msg = self.log.warning
else:
msg = self.log.debug
msg("Config file not found, skipping: %s", config_file_name)
except Exception:
# For testing purposes.
if not suppress_errors:
raise
self.log.warning("Error loading config file: %s" %
self.config_file_name, exc_info=True)
def init_profile_dir(self):
"""initialize the profile dir"""
self._in_init_profile_dir = True
if self.profile_dir is not None:
# already ran
return
if 'ProfileDir.location' not in self.config:
# location not specified, find by profile name
try:
p = ProfileDir.find_profile_dir_by_name(self.ipython_dir, self.profile, self.config)
except ProfileDirError:
# not found, maybe create it (always create default profile)
if self.auto_create or self.profile == 'default':
try:
p = ProfileDir.create_profile_dir_by_name(self.ipython_dir, self.profile, self.config)
except ProfileDirError:
self.log.fatal("Could not create profile: %r"%self.profile)
self.exit(1)
else:
self.log.info("Created profile dir: %r"%p.location)
else:
self.log.fatal("Profile %r not found."%self.profile)
self.exit(1)
else:
self.log.debug(f"Using existing profile dir: {p.location!r}")
else:
location = self.config.ProfileDir.location
# location is fully specified
try:
p = ProfileDir.find_profile_dir(location, self.config)
except ProfileDirError:
# not found, maybe create it
if self.auto_create:
try:
p = ProfileDir.create_profile_dir(location, self.config)
except ProfileDirError:
self.log.fatal("Could not create profile directory: %r"%location)
self.exit(1)
else:
self.log.debug("Creating new profile dir: %r"%location)
else:
self.log.fatal("Profile directory %r not found."%location)
self.exit(1)
else:
self.log.debug(f"Using existing profile dir: {p.location!r}")
# if profile_dir is specified explicitly, set profile name
dir_name = os.path.basename(p.location)
if dir_name.startswith('profile_'):
self.profile = dir_name[8:]
self.profile_dir = p
self.config_file_paths.append(p.location)
self._in_init_profile_dir = False
def init_config_files(self):
"""[optionally] copy default config files into profile dir."""
self.config_file_paths.extend(ENV_CONFIG_DIRS)
self.config_file_paths.extend(SYSTEM_CONFIG_DIRS)
# copy config files
path = Path(self.builtin_profile_dir)
if self.copy_config_files:
src = self.profile
cfg = self.config_file_name
if path and (path / cfg).exists():
self.log.warning(
"Staging %r from %s into %r [overwrite=%s]"
% (cfg, src, self.profile_dir.location, self.overwrite)
)
self.profile_dir.copy_config_file(cfg, path=path, overwrite=self.overwrite)
else:
self.stage_default_config_file()
else:
# Still stage *bundled* config files, but not generated ones
# This is necessary for `ipython profile=sympy` to load the profile
# on the first go
files = path.glob("*.py")
for fullpath in files:
cfg = fullpath.name
if self.profile_dir.copy_config_file(cfg, path=path, overwrite=False):
# file was copied
self.log.warning("Staging bundled %s from %s into %r"%(
cfg, self.profile, self.profile_dir.location)
)
def stage_default_config_file(self):
"""auto generate default config file, and stage it into the profile."""
s = self.generate_config_file()
config_file = Path(self.profile_dir.location) / self.config_file_name
if self.overwrite or not config_file.exists():
self.log.warning("Generating default config file: %r" % (config_file))
config_file.write_text(s, encoding="utf-8")
@catch_config_error
def initialize(self, argv=None):
# don't hook up crash handler before parsing command-line
self.parse_command_line(argv)
self.init_crash_handler()
if self.subapp is not None:
# stop here if subapp is taking over
return
# save a copy of CLI config to re-load after config files
# so that it has highest priority
cl_config = deepcopy(self.config)
self.init_profile_dir()
self.init_config_files()
self.load_config_file()
# enforce cl-opts override configfile opts:
self.update_config(cl_config)

@ -0,0 +1,156 @@
"""
Async helper function that are invalid syntax on Python 3.5 and below.
This code is best effort, and may have edge cases not behaving as expected. In
particular it contain a number of heuristics to detect whether code is
effectively async and need to run in an event loop or not.
Some constructs (like top-level `return`, or `yield`) are taken care of
explicitly to actually raise a SyntaxError and stay as close as possible to
Python semantics.
"""
import ast
import asyncio
import inspect
from functools import wraps
_asyncio_event_loop = None
def get_asyncio_loop():
"""asyncio has deprecated get_event_loop
Replicate it here, with our desired semantics:
- always returns a valid, not-closed loop
- not thread-local like asyncio's,
because we only want one loop for IPython
- if called from inside a coroutine (e.g. in ipykernel),
return the running loop
.. versionadded:: 8.0
"""
try:
return asyncio.get_running_loop()
except RuntimeError:
# not inside a coroutine,
# track our own global
pass
# not thread-local like asyncio's,
# because we only track one event loop to run for IPython itself,
# always in the main thread.
global _asyncio_event_loop
if _asyncio_event_loop is None or _asyncio_event_loop.is_closed():
_asyncio_event_loop = asyncio.new_event_loop()
return _asyncio_event_loop
class _AsyncIORunner:
def __call__(self, coro):
"""
Handler for asyncio autoawait
"""
return get_asyncio_loop().run_until_complete(coro)
def __str__(self):
return "asyncio"
_asyncio_runner = _AsyncIORunner()
class _AsyncIOProxy:
"""Proxy-object for an asyncio
Any coroutine methods will be wrapped in event_loop.run_
"""
def __init__(self, obj, event_loop):
self._obj = obj
self._event_loop = event_loop
def __repr__(self):
return f"<_AsyncIOProxy({self._obj!r})>"
def __getattr__(self, key):
attr = getattr(self._obj, key)
if inspect.iscoroutinefunction(attr):
# if it's a coroutine method,
# return a threadsafe wrapper onto the _current_ asyncio loop
@wraps(attr)
def _wrapped(*args, **kwargs):
concurrent_future = asyncio.run_coroutine_threadsafe(
attr(*args, **kwargs), self._event_loop
)
return asyncio.wrap_future(concurrent_future)
return _wrapped
else:
return attr
def __dir__(self):
return dir(self._obj)
def _curio_runner(coroutine):
"""
handler for curio autoawait
"""
import curio
return curio.run(coroutine)
def _trio_runner(async_fn):
import trio
async def loc(coro):
"""
We need the dummy no-op async def to protect from
trio's internal. See https://github.com/python-trio/trio/issues/89
"""
return await coro
return trio.run(loc, async_fn)
def _pseudo_sync_runner(coro):
"""
A runner that does not really allow async execution, and just advance the coroutine.
See discussion in https://github.com/python-trio/trio/issues/608,
Credit to Nathaniel Smith
"""
try:
coro.send(None)
except StopIteration as exc:
return exc.value
else:
# TODO: do not raise but return an execution result with the right info.
raise RuntimeError(
"{coro_name!r} needs a real async loop".format(coro_name=coro.__name__)
)
def _should_be_async(cell: str) -> bool:
"""Detect if a block of code need to be wrapped in an `async def`
Attempt to parse the block of code, it it compile we're fine.
Otherwise we wrap if and try to compile.
If it works, assume it should be async. Otherwise Return False.
Not handled yet: If the block of code has a return statement as the top
level, it will be seen as async. This is a know limitation.
"""
try:
code = compile(
cell, "<>", "exec", flags=getattr(ast, "PyCF_ALLOW_TOP_LEVEL_AWAIT", 0x0)
)
return inspect.CO_COROUTINE & code.co_flags == inspect.CO_COROUTINE
except (SyntaxError, MemoryError):
return False

@ -0,0 +1,70 @@
# encoding: utf-8
"""
Autocall capabilities for IPython.core.
Authors:
* Brian Granger
* Fernando Perez
* Thomas Kluyver
Notes
-----
"""
#-----------------------------------------------------------------------------
# Copyright (C) 2008-2011 The IPython Development Team
#
# Distributed under the terms of the BSD License. The full license is in
# the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------
#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
#-----------------------------------------------------------------------------
# Code
#-----------------------------------------------------------------------------
class IPyAutocall(object):
""" Instances of this class are always autocalled
This happens regardless of 'autocall' variable state. Use this to
develop macro-like mechanisms.
"""
_ip = None
rewrite = True
def __init__(self, ip=None):
self._ip = ip
def set_ip(self, ip):
"""Will be used to set _ip point to current ipython instance b/f call
Override this method if you don't want this to happen.
"""
self._ip = ip
class ExitAutocall(IPyAutocall):
"""An autocallable object which will be added to the user namespace so that
exit, exit(), quit or quit() are all valid ways to close the shell."""
rewrite = False
def __call__(self):
self._ip.ask_exit()
class ZMQExitAutocall(ExitAutocall):
"""Exit IPython. Autocallable, so it needn't be explicitly called.
Parameters
----------
keep_kernel : bool
If True, leave the kernel alive. Otherwise, tell the kernel to exit too
(default).
"""
def __call__(self, keep_kernel=False):
self._ip.keepkernel_on_exit = keep_kernel
self._ip.ask_exit()

@ -0,0 +1,86 @@
"""
A context manager for managing things injected into :mod:`builtins`.
"""
# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.
import builtins as builtin_mod
from traitlets.config.configurable import Configurable
from traitlets import Instance
class __BuiltinUndefined(object): pass
BuiltinUndefined = __BuiltinUndefined()
class __HideBuiltin(object): pass
HideBuiltin = __HideBuiltin()
class BuiltinTrap(Configurable):
shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
allow_none=True)
def __init__(self, shell=None):
super(BuiltinTrap, self).__init__(shell=shell, config=None)
self._orig_builtins = {}
# We define this to track if a single BuiltinTrap is nested.
# Only turn off the trap when the outermost call to __exit__ is made.
self._nested_level = 0
self.shell = shell
# builtins we always add - if set to HideBuiltin, they will just
# be removed instead of being replaced by something else
self.auto_builtins = {'exit': HideBuiltin,
'quit': HideBuiltin,
'get_ipython': self.shell.get_ipython,
}
def __enter__(self):
if self._nested_level == 0:
self.activate()
self._nested_level += 1
# I return self, so callers can use add_builtin in a with clause.
return self
def __exit__(self, type, value, traceback):
if self._nested_level == 1:
self.deactivate()
self._nested_level -= 1
# Returning False will cause exceptions to propagate
return False
def add_builtin(self, key, value):
"""Add a builtin and save the original."""
bdict = builtin_mod.__dict__
orig = bdict.get(key, BuiltinUndefined)
if value is HideBuiltin:
if orig is not BuiltinUndefined: #same as 'key in bdict'
self._orig_builtins[key] = orig
del bdict[key]
else:
self._orig_builtins[key] = orig
bdict[key] = value
def remove_builtin(self, key, orig):
"""Remove an added builtin and re-set the original."""
if orig is BuiltinUndefined:
del builtin_mod.__dict__[key]
else:
builtin_mod.__dict__[key] = orig
def activate(self):
"""Store ipython references in the __builtin__ namespace."""
add_builtin = self.add_builtin
for name, func in self.auto_builtins.items():
add_builtin(name, func)
def deactivate(self):
"""Remove any builtins which might have been added by add_builtins, or
restore overwritten ones to their previous values."""
remove_builtin = self.remove_builtin
for key, val in self._orig_builtins.items():
remove_builtin(key, val)
self._orig_builtins.clear()
self._builtins_added = False

@ -0,0 +1,196 @@
"""Compiler tools with improved interactive support.
Provides compilation machinery similar to codeop, but with caching support so
we can provide interactive tracebacks.
Authors
-------
* Robert Kern
* Fernando Perez
* Thomas Kluyver
"""
# Note: though it might be more natural to name this module 'compiler', that
# name is in the stdlib and name collisions with the stdlib tend to produce
# weird problems (often with third-party tools).
#-----------------------------------------------------------------------------
# Copyright (C) 2010-2011 The IPython Development Team.
#
# Distributed under the terms of the BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
# Stdlib imports
import __future__
from ast import PyCF_ONLY_AST
import codeop
import functools
import hashlib
import linecache
import operator
import time
from contextlib import contextmanager
#-----------------------------------------------------------------------------
# Constants
#-----------------------------------------------------------------------------
# Roughly equal to PyCF_MASK | PyCF_MASK_OBSOLETE as defined in pythonrun.h,
# this is used as a bitmask to extract future-related code flags.
PyCF_MASK = functools.reduce(operator.or_,
(getattr(__future__, fname).compiler_flag
for fname in __future__.all_feature_names))
#-----------------------------------------------------------------------------
# Local utilities
#-----------------------------------------------------------------------------
def code_name(code, number=0):
""" Compute a (probably) unique name for code for caching.
This now expects code to be unicode.
"""
hash_digest = hashlib.sha1(code.encode("utf-8")).hexdigest()
# Include the number and 12 characters of the hash in the name. It's
# pretty much impossible that in a single session we'll have collisions
# even with truncated hashes, and the full one makes tracebacks too long
return '<ipython-input-{0}-{1}>'.format(number, hash_digest[:12])
#-----------------------------------------------------------------------------
# Classes and functions
#-----------------------------------------------------------------------------
class CachingCompiler(codeop.Compile):
"""A compiler that caches code compiled from interactive statements.
"""
def __init__(self):
codeop.Compile.__init__(self)
# This is ugly, but it must be done this way to allow multiple
# simultaneous ipython instances to coexist. Since Python itself
# directly accesses the data structures in the linecache module, and
# the cache therein is global, we must work with that data structure.
# We must hold a reference to the original checkcache routine and call
# that in our own check_cache() below, but the special IPython cache
# must also be shared by all IPython instances. If we were to hold
# separate caches (one in each CachingCompiler instance), any call made
# by Python itself to linecache.checkcache() would obliterate the
# cached data from the other IPython instances.
if not hasattr(linecache, '_ipython_cache'):
linecache._ipython_cache = {}
if not hasattr(linecache, '_checkcache_ori'):
linecache._checkcache_ori = linecache.checkcache
# Now, we must monkeypatch the linecache directly so that parts of the
# stdlib that call it outside our control go through our codepath
# (otherwise we'd lose our tracebacks).
linecache.checkcache = check_linecache_ipython
# Caching a dictionary { filename: execution_count } for nicely
# rendered tracebacks. The filename corresponds to the filename
# argument used for the builtins.compile function.
self._filename_map = {}
def ast_parse(self, source, filename='<unknown>', symbol='exec'):
"""Parse code to an AST with the current compiler flags active.
Arguments are exactly the same as ast.parse (in the standard library),
and are passed to the built-in compile function."""
return compile(source, filename, symbol, self.flags | PyCF_ONLY_AST, 1)
def reset_compiler_flags(self):
"""Reset compiler flags to default state."""
# This value is copied from codeop.Compile.__init__, so if that ever
# changes, it will need to be updated.
self.flags = codeop.PyCF_DONT_IMPLY_DEDENT
@property
def compiler_flags(self):
"""Flags currently active in the compilation process.
"""
return self.flags
def get_code_name(self, raw_code, transformed_code, number):
"""Compute filename given the code, and the cell number.
Parameters
----------
raw_code : str
The raw cell code.
transformed_code : str
The executable Python source code to cache and compile.
number : int
A number which forms part of the code's name. Used for the execution
counter.
Returns
-------
The computed filename.
"""
return code_name(transformed_code, number)
def cache(self, transformed_code, number=0, raw_code=None):
"""Make a name for a block of code, and cache the code.
Parameters
----------
transformed_code : str
The executable Python source code to cache and compile.
number : int
A number which forms part of the code's name. Used for the execution
counter.
raw_code : str
The raw code before transformation, if None, set to `transformed_code`.
Returns
-------
The name of the cached code (as a string). Pass this as the filename
argument to compilation, so that tracebacks are correctly hooked up.
"""
if raw_code is None:
raw_code = transformed_code
name = self.get_code_name(raw_code, transformed_code, number)
# Save the execution count
self._filename_map[name] = number
entry = (
len(transformed_code),
time.time(),
[line + "\n" for line in transformed_code.splitlines()],
name,
)
linecache.cache[name] = entry
linecache._ipython_cache[name] = entry
return name
@contextmanager
def extra_flags(self, flags):
## bits that we'll set to 1
turn_on_bits = ~self.flags & flags
self.flags = self.flags | flags
try:
yield
finally:
# turn off only the bits we turned on so that something like
# __future__ that set flags stays.
self.flags &= ~turn_on_bits
def check_linecache_ipython(*args):
"""Call linecache.checkcache() safely protecting our cached values.
"""
# First call the original checkcache as intended
linecache._checkcache_ori(*args)
# Then, update back the cache with our data, so that tracebacks related
# to our compiled codes can be produced.
linecache.cache.update(linecache._ipython_cache)

@ -0,0 +1,370 @@
# encoding: utf-8
"""Implementations for various useful completers.
These are all loaded by default by IPython.
"""
#-----------------------------------------------------------------------------
# Copyright (C) 2010-2011 The IPython Development Team.
#
# Distributed under the terms of the BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
# Stdlib imports
import glob
import inspect
import os
import re
import sys
from importlib import import_module
from importlib.machinery import all_suffixes
# Third-party imports
from time import time
from zipimport import zipimporter
# Our own imports
from .completer import expand_user, compress_user
from .error import TryNext
from ..utils._process_common import arg_split
# FIXME: this should be pulled in with the right call via the component system
from IPython import get_ipython
from typing import List
#-----------------------------------------------------------------------------
# Globals and constants
#-----------------------------------------------------------------------------
_suffixes = all_suffixes()
# Time in seconds after which the rootmodules will be stored permanently in the
# ipython ip.db database (kept in the user's .ipython dir).
TIMEOUT_STORAGE = 2
# Time in seconds after which we give up
TIMEOUT_GIVEUP = 20
# Regular expression for the python import statement
import_re = re.compile(r'(?P<name>[^\W\d]\w*?)'
r'(?P<package>[/\\]__init__)?'
r'(?P<suffix>%s)$' %
r'|'.join(re.escape(s) for s in _suffixes))
# RE for the ipython %run command (python + ipython scripts)
magic_run_re = re.compile(r'.*(\.ipy|\.ipynb|\.py[w]?)$')
#-----------------------------------------------------------------------------
# Local utilities
#-----------------------------------------------------------------------------
def module_list(path):
"""
Return the list containing the names of the modules available in the given
folder.
"""
# sys.path has the cwd as an empty string, but isdir/listdir need it as '.'
if path == '':
path = '.'
# A few local constants to be used in loops below
pjoin = os.path.join
if os.path.isdir(path):
# Build a list of all files in the directory and all files
# in its subdirectories. For performance reasons, do not
# recurse more than one level into subdirectories.
files = []
for root, dirs, nondirs in os.walk(path, followlinks=True):
subdir = root[len(path)+1:]
if subdir:
files.extend(pjoin(subdir, f) for f in nondirs)
dirs[:] = [] # Do not recurse into additional subdirectories.
else:
files.extend(nondirs)
else:
try:
files = list(zipimporter(path)._files.keys())
except:
files = []
# Build a list of modules which match the import_re regex.
modules = []
for f in files:
m = import_re.match(f)
if m:
modules.append(m.group('name'))
return list(set(modules))
def get_root_modules():
"""
Returns a list containing the names of all the modules available in the
folders of the pythonpath.
ip.db['rootmodules_cache'] maps sys.path entries to list of modules.
"""
ip = get_ipython()
if ip is None:
# No global shell instance to store cached list of modules.
# Don't try to scan for modules every time.
return list(sys.builtin_module_names)
rootmodules_cache = ip.db.get('rootmodules_cache', {})
rootmodules = list(sys.builtin_module_names)
start_time = time()
store = False
for path in sys.path:
try:
modules = rootmodules_cache[path]
except KeyError:
modules = module_list(path)
try:
modules.remove('__init__')
except ValueError:
pass
if path not in ('', '.'): # cwd modules should not be cached
rootmodules_cache[path] = modules
if time() - start_time > TIMEOUT_STORAGE and not store:
store = True
print("\nCaching the list of root modules, please wait!")
print("(This will only be done once - type '%rehashx' to "
"reset cache!)\n")
sys.stdout.flush()
if time() - start_time > TIMEOUT_GIVEUP:
print("This is taking too long, we give up.\n")
return []
rootmodules.extend(modules)
if store:
ip.db['rootmodules_cache'] = rootmodules_cache
rootmodules = list(set(rootmodules))
return rootmodules
def is_importable(module, attr, only_modules):
if only_modules:
return inspect.ismodule(getattr(module, attr))
else:
return not(attr[:2] == '__' and attr[-2:] == '__')
def is_possible_submodule(module, attr):
try:
obj = getattr(module, attr)
except AttributeError:
# Is possilby an unimported submodule
return True
except TypeError:
# https://github.com/ipython/ipython/issues/9678
return False
return inspect.ismodule(obj)
def try_import(mod: str, only_modules=False) -> List[str]:
"""
Try to import given module and return list of potential completions.
"""
mod = mod.rstrip('.')
try:
m = import_module(mod)
except:
return []
m_is_init = '__init__' in (getattr(m, '__file__', '') or '')
completions = []
if (not hasattr(m, '__file__')) or (not only_modules) or m_is_init:
completions.extend( [attr for attr in dir(m) if
is_importable(m, attr, only_modules)])
m_all = getattr(m, "__all__", [])
if only_modules:
completions.extend(attr for attr in m_all if is_possible_submodule(m, attr))
else:
completions.extend(m_all)
if m_is_init:
completions.extend(module_list(os.path.dirname(m.__file__)))
completions_set = {c for c in completions if isinstance(c, str)}
completions_set.discard('__init__')
return list(completions_set)
#-----------------------------------------------------------------------------
# Completion-related functions.
#-----------------------------------------------------------------------------
def quick_completer(cmd, completions):
r""" Easily create a trivial completer for a command.
Takes either a list of completions, or all completions in string (that will
be split on whitespace).
Example::
[d:\ipython]|1> import ipy_completers
[d:\ipython]|2> ipy_completers.quick_completer('foo', ['bar','baz'])
[d:\ipython]|3> foo b<TAB>
bar baz
[d:\ipython]|3> foo ba
"""
if isinstance(completions, str):
completions = completions.split()
def do_complete(self, event):
return completions
get_ipython().set_hook('complete_command',do_complete, str_key = cmd)
def module_completion(line):
"""
Returns a list containing the completion possibilities for an import line.
The line looks like this :
'import xml.d'
'from xml.dom import'
"""
words = line.split(' ')
nwords = len(words)
# from whatever <tab> -> 'import '
if nwords == 3 and words[0] == 'from':
return ['import ']
# 'from xy<tab>' or 'import xy<tab>'
if nwords < 3 and (words[0] in {'%aimport', 'import', 'from'}) :
if nwords == 1:
return get_root_modules()
mod = words[1].split('.')
if len(mod) < 2:
return get_root_modules()
completion_list = try_import('.'.join(mod[:-1]), True)
return ['.'.join(mod[:-1] + [el]) for el in completion_list]
# 'from xyz import abc<tab>'
if nwords >= 3 and words[0] == 'from':
mod = words[1]
return try_import(mod)
#-----------------------------------------------------------------------------
# Completers
#-----------------------------------------------------------------------------
# These all have the func(self, event) signature to be used as custom
# completers
def module_completer(self,event):
"""Give completions after user has typed 'import ...' or 'from ...'"""
# This works in all versions of python. While 2.5 has
# pkgutil.walk_packages(), that particular routine is fairly dangerous,
# since it imports *EVERYTHING* on sys.path. That is: a) very slow b) full
# of possibly problematic side effects.
# This search the folders in the sys.path for available modules.
return module_completion(event.line)
# FIXME: there's a lot of logic common to the run, cd and builtin file
# completers, that is currently reimplemented in each.
def magic_run_completer(self, event):
"""Complete files that end in .py or .ipy or .ipynb for the %run command.
"""
comps = arg_split(event.line, strict=False)
# relpath should be the current token that we need to complete.
if (len(comps) > 1) and (not event.line.endswith(' ')):
relpath = comps[-1].strip("'\"")
else:
relpath = ''
#print("\nev=", event) # dbg
#print("rp=", relpath) # dbg
#print('comps=', comps) # dbg
lglob = glob.glob
isdir = os.path.isdir
relpath, tilde_expand, tilde_val = expand_user(relpath)
# Find if the user has already typed the first filename, after which we
# should complete on all files, since after the first one other files may
# be arguments to the input script.
if any(magic_run_re.match(c) for c in comps):
matches = [f.replace('\\','/') + ('/' if isdir(f) else '')
for f in lglob(relpath+'*')]
else:
dirs = [f.replace('\\','/') + "/" for f in lglob(relpath+'*') if isdir(f)]
pys = [f.replace('\\','/')
for f in lglob(relpath+'*.py') + lglob(relpath+'*.ipy') +
lglob(relpath+'*.ipynb') + lglob(relpath + '*.pyw')]
matches = dirs + pys
#print('run comp:', dirs+pys) # dbg
return [compress_user(p, tilde_expand, tilde_val) for p in matches]
def cd_completer(self, event):
"""Completer function for cd, which only returns directories."""
ip = get_ipython()
relpath = event.symbol
#print(event) # dbg
if event.line.endswith('-b') or ' -b ' in event.line:
# return only bookmark completions
bkms = self.db.get('bookmarks', None)
if bkms:
return bkms.keys()
else:
return []
if event.symbol == '-':
width_dh = str(len(str(len(ip.user_ns['_dh']) + 1)))
# jump in directory history by number
fmt = '-%0' + width_dh +'d [%s]'
ents = [ fmt % (i,s) for i,s in enumerate(ip.user_ns['_dh'])]
if len(ents) > 1:
return ents
return []
if event.symbol.startswith('--'):
return ["--" + os.path.basename(d) for d in ip.user_ns['_dh']]
# Expand ~ in path and normalize directory separators.
relpath, tilde_expand, tilde_val = expand_user(relpath)
relpath = relpath.replace('\\','/')
found = []
for d in [f.replace('\\','/') + '/' for f in glob.glob(relpath+'*')
if os.path.isdir(f)]:
if ' ' in d:
# we don't want to deal with any of that, complex code
# for this is elsewhere
raise TryNext
found.append(d)
if not found:
if os.path.isdir(relpath):
return [compress_user(relpath, tilde_expand, tilde_val)]
# if no completions so far, try bookmarks
bks = self.db.get('bookmarks',{})
bkmatches = [s for s in bks if s.startswith(event.symbol)]
if bkmatches:
return bkmatches
raise TryNext
return [compress_user(p, tilde_expand, tilde_val) for p in found]
def reset_completer(self, event):
"A completer for %reset magic"
return '-f -s in out array dhist'.split()

@ -0,0 +1,237 @@
# encoding: utf-8
"""sys.excepthook for IPython itself, leaves a detailed report on disk.
Authors:
* Fernando Perez
* Brian E. Granger
"""
#-----------------------------------------------------------------------------
# Copyright (C) 2001-2007 Fernando Perez. <fperez@colorado.edu>
# Copyright (C) 2008-2011 The IPython Development Team
#
# Distributed under the terms of the BSD License. The full license is in
# the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------
#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
import os
import sys
import traceback
from pprint import pformat
from pathlib import Path
from IPython.core import ultratb
from IPython.core.release import author_email
from IPython.utils.sysinfo import sys_info
from IPython.utils.py3compat import input
from IPython.core.release import __version__ as version
from typing import Optional
#-----------------------------------------------------------------------------
# Code
#-----------------------------------------------------------------------------
# Template for the user message.
_default_message_template = """\
Oops, {app_name} crashed. We do our best to make it stable, but...
A crash report was automatically generated with the following information:
- A verbatim copy of the crash traceback.
- A copy of your input history during this session.
- Data on your current {app_name} configuration.
It was left in the file named:
\t'{crash_report_fname}'
If you can email this file to the developers, the information in it will help
them in understanding and correcting the problem.
You can mail it to: {contact_name} at {contact_email}
with the subject '{app_name} Crash Report'.
If you want to do it now, the following command will work (under Unix):
mail -s '{app_name} Crash Report' {contact_email} < {crash_report_fname}
In your email, please also include information about:
- The operating system under which the crash happened: Linux, macOS, Windows,
other, and which exact version (for example: Ubuntu 16.04.3, macOS 10.13.2,
Windows 10 Pro), and whether it is 32-bit or 64-bit;
- How {app_name} was installed: using pip or conda, from GitHub, as part of
a Docker container, or other, providing more detail if possible;
- How to reproduce the crash: what exact sequence of instructions can one
input to get the same crash? Ideally, find a minimal yet complete sequence
of instructions that yields the crash.
To ensure accurate tracking of this issue, please file a report about it at:
{bug_tracker}
"""
_lite_message_template = """
If you suspect this is an IPython {version} bug, please report it at:
https://github.com/ipython/ipython/issues
or send an email to the mailing list at {email}
You can print a more detailed traceback right now with "%tb", or use "%debug"
to interactively debug it.
Extra-detailed tracebacks for bug-reporting purposes can be enabled via:
{config}Application.verbose_crash=True
"""
class CrashHandler(object):
"""Customizable crash handlers for IPython applications.
Instances of this class provide a :meth:`__call__` method which can be
used as a ``sys.excepthook``. The :meth:`__call__` signature is::
def __call__(self, etype, evalue, etb)
"""
message_template = _default_message_template
section_sep = '\n\n'+'*'*75+'\n\n'
def __init__(
self,
app,
contact_name: Optional[str] = None,
contact_email: Optional[str] = None,
bug_tracker: Optional[str] = None,
show_crash_traceback: bool = True,
call_pdb: bool = False,
):
"""Create a new crash handler
Parameters
----------
app : Application
A running :class:`Application` instance, which will be queried at
crash time for internal information.
contact_name : str
A string with the name of the person to contact.
contact_email : str
A string with the email address of the contact.
bug_tracker : str
A string with the URL for your project's bug tracker.
show_crash_traceback : bool
If false, don't print the crash traceback on stderr, only generate
the on-disk report
call_pdb
Whether to call pdb on crash
Attributes
----------
These instances contain some non-argument attributes which allow for
further customization of the crash handler's behavior. Please see the
source for further details.
"""
self.crash_report_fname = "Crash_report_%s.txt" % app.name
self.app = app
self.call_pdb = call_pdb
#self.call_pdb = True # dbg
self.show_crash_traceback = show_crash_traceback
self.info = dict(app_name = app.name,
contact_name = contact_name,
contact_email = contact_email,
bug_tracker = bug_tracker,
crash_report_fname = self.crash_report_fname)
def __call__(self, etype, evalue, etb):
"""Handle an exception, call for compatible with sys.excepthook"""
# do not allow the crash handler to be called twice without reinstalling it
# this prevents unlikely errors in the crash handling from entering an
# infinite loop.
sys.excepthook = sys.__excepthook__
# Report tracebacks shouldn't use color in general (safer for users)
color_scheme = 'NoColor'
# Use this ONLY for developer debugging (keep commented out for release)
#color_scheme = 'Linux' # dbg
try:
rptdir = self.app.ipython_dir
except:
rptdir = Path.cwd()
if rptdir is None or not Path.is_dir(rptdir):
rptdir = Path.cwd()
report_name = rptdir / self.crash_report_fname
# write the report filename into the instance dict so it can get
# properly expanded out in the user message template
self.crash_report_fname = report_name
self.info['crash_report_fname'] = report_name
TBhandler = ultratb.VerboseTB(
color_scheme=color_scheme,
long_header=1,
call_pdb=self.call_pdb,
)
if self.call_pdb:
TBhandler(etype,evalue,etb)
return
else:
traceback = TBhandler.text(etype,evalue,etb,context=31)
# print traceback to screen
if self.show_crash_traceback:
print(traceback, file=sys.stderr)
# and generate a complete report on disk
try:
report = open(report_name, "w", encoding="utf-8")
except:
print('Could not create crash report on disk.', file=sys.stderr)
return
with report:
# Inform user on stderr of what happened
print('\n'+'*'*70+'\n', file=sys.stderr)
print(self.message_template.format(**self.info), file=sys.stderr)
# Construct report on disk
report.write(self.make_report(traceback))
input("Hit <Enter> to quit (your terminal may close):")
def make_report(self,traceback):
"""Return a string containing a crash report."""
sec_sep = self.section_sep
report = ['*'*75+'\n\n'+'IPython post-mortem report\n\n']
rpt_add = report.append
rpt_add(sys_info())
try:
config = pformat(self.app.config)
rpt_add(sec_sep)
rpt_add('Application name: %s\n\n' % self.app_name)
rpt_add('Current user configuration structure:\n\n')
rpt_add(config)
except:
pass
rpt_add(sec_sep+'Crash traceback:\n\n' + traceback)
return ''.join(report)
def crash_handler_lite(etype, evalue, tb):
"""a light excepthook, adding a small message to the usual traceback"""
traceback.print_exception(etype, evalue, tb)
from IPython.core.interactiveshell import InteractiveShell
if InteractiveShell.initialized():
# we are in a Shell environment, give %magic example
config = "%config "
else:
# we are not in a shell, show generic config
config = "c."
print(_lite_message_template.format(email=author_email, config=config, version=version), file=sys.stderr)

@ -0,0 +1,999 @@
# -*- coding: utf-8 -*-
"""
Pdb debugger class.
This is an extension to PDB which adds a number of new features.
Note that there is also the `IPython.terminal.debugger` class which provides UI
improvements.
We also strongly recommend to use this via the `ipdb` package, which provides
extra configuration options.
Among other things, this subclass of PDB:
- supports many IPython magics like pdef/psource
- hide frames in tracebacks based on `__tracebackhide__`
- allows to skip frames based on `__debuggerskip__`
The skipping and hiding frames are configurable via the `skip_predicates`
command.
By default, frames from readonly files will be hidden, frames containing
``__tracebackhide__=True`` will be hidden.
Frames containing ``__debuggerskip__`` will be stepped over, frames who's parent
frames value of ``__debuggerskip__`` is ``True`` will be skipped.
>>> def helpers_helper():
... pass
...
... def helper_1():
... print("don't step in me")
... helpers_helpers() # will be stepped over unless breakpoint set.
...
...
... def helper_2():
... print("in me neither")
...
One can define a decorator that wraps a function between the two helpers:
>>> def pdb_skipped_decorator(function):
...
...
... def wrapped_fn(*args, **kwargs):
... __debuggerskip__ = True
... helper_1()
... __debuggerskip__ = False
... result = function(*args, **kwargs)
... __debuggerskip__ = True
... helper_2()
... # setting __debuggerskip__ to False again is not necessary
... return result
...
... return wrapped_fn
When decorating a function, ipdb will directly step into ``bar()`` by
default:
>>> @foo_decorator
... def bar(x, y):
... return x * y
You can toggle the behavior with
ipdb> skip_predicates debuggerskip false
or configure it in your ``.pdbrc``
License
-------
Modified from the standard pdb.Pdb class to avoid including readline, so that
the command line completion of other programs which include this isn't
damaged.
In the future, this class will be expanded with improvements over the standard
pdb.
The original code in this file is mainly lifted out of cmd.py in Python 2.2,
with minor changes. Licensing should therefore be under the standard Python
terms. For details on the PSF (Python Software Foundation) standard license,
see:
https://docs.python.org/2/license.html
All the changes since then are under the same license as IPython.
"""
#*****************************************************************************
#
# This file is licensed under the PSF license.
#
# Copyright (C) 2001 Python Software Foundation, www.python.org
# Copyright (C) 2005-2006 Fernando Perez. <fperez@colorado.edu>
#
#
#*****************************************************************************
import inspect
import linecache
import sys
import warnings
import re
import os
from IPython import get_ipython
from IPython.utils import PyColorize
from IPython.utils import coloransi, py3compat
from IPython.core.excolors import exception_colors
# skip module docstests
__skip_doctest__ = True
prompt = 'ipdb> '
# We have to check this directly from sys.argv, config struct not yet available
from pdb import Pdb as OldPdb
# Allow the set_trace code to operate outside of an ipython instance, even if
# it does so with some limitations. The rest of this support is implemented in
# the Tracer constructor.
DEBUGGERSKIP = "__debuggerskip__"
def make_arrow(pad):
"""generate the leading arrow in front of traceback or debugger"""
if pad >= 2:
return '-'*(pad-2) + '> '
elif pad == 1:
return '>'
return ''
def BdbQuit_excepthook(et, ev, tb, excepthook=None):
"""Exception hook which handles `BdbQuit` exceptions.
All other exceptions are processed using the `excepthook`
parameter.
"""
raise ValueError(
"`BdbQuit_excepthook` is deprecated since version 5.1",
)
def BdbQuit_IPython_excepthook(self, et, ev, tb, tb_offset=None):
raise ValueError(
"`BdbQuit_IPython_excepthook` is deprecated since version 5.1",
DeprecationWarning, stacklevel=2)
RGX_EXTRA_INDENT = re.compile(r'(?<=\n)\s+')
def strip_indentation(multiline_string):
return RGX_EXTRA_INDENT.sub('', multiline_string)
def decorate_fn_with_doc(new_fn, old_fn, additional_text=""):
"""Make new_fn have old_fn's doc string. This is particularly useful
for the ``do_...`` commands that hook into the help system.
Adapted from from a comp.lang.python posting
by Duncan Booth."""
def wrapper(*args, **kw):
return new_fn(*args, **kw)
if old_fn.__doc__:
wrapper.__doc__ = strip_indentation(old_fn.__doc__) + additional_text
return wrapper
class Pdb(OldPdb):
"""Modified Pdb class, does not load readline.
for a standalone version that uses prompt_toolkit, see
`IPython.terminal.debugger.TerminalPdb` and
`IPython.terminal.debugger.set_trace()`
This debugger can hide and skip frames that are tagged according to some predicates.
See the `skip_predicates` commands.
"""
default_predicates = {
"tbhide": True,
"readonly": False,
"ipython_internal": True,
"debuggerskip": True,
}
def __init__(self, completekey=None, stdin=None, stdout=None, context=5, **kwargs):
"""Create a new IPython debugger.
Parameters
----------
completekey : default None
Passed to pdb.Pdb.
stdin : default None
Passed to pdb.Pdb.
stdout : default None
Passed to pdb.Pdb.
context : int
Number of lines of source code context to show when
displaying stacktrace information.
**kwargs
Passed to pdb.Pdb.
Notes
-----
The possibilities are python version dependent, see the python
docs for more info.
"""
# Parent constructor:
try:
self.context = int(context)
if self.context <= 0:
raise ValueError("Context must be a positive integer")
except (TypeError, ValueError) as e:
raise ValueError("Context must be a positive integer") from e
# `kwargs` ensures full compatibility with stdlib's `pdb.Pdb`.
OldPdb.__init__(self, completekey, stdin, stdout, **kwargs)
# IPython changes...
self.shell = get_ipython()
if self.shell is None:
save_main = sys.modules['__main__']
# No IPython instance running, we must create one
from IPython.terminal.interactiveshell import \
TerminalInteractiveShell
self.shell = TerminalInteractiveShell.instance()
# needed by any code which calls __import__("__main__") after
# the debugger was entered. See also #9941.
sys.modules["__main__"] = save_main
color_scheme = self.shell.colors
self.aliases = {}
# Create color table: we copy the default one from the traceback
# module and add a few attributes needed for debugging
self.color_scheme_table = exception_colors()
# shorthands
C = coloransi.TermColors
cst = self.color_scheme_table
cst['NoColor'].colors.prompt = C.NoColor
cst['NoColor'].colors.breakpoint_enabled = C.NoColor
cst['NoColor'].colors.breakpoint_disabled = C.NoColor
cst['Linux'].colors.prompt = C.Green
cst['Linux'].colors.breakpoint_enabled = C.LightRed
cst['Linux'].colors.breakpoint_disabled = C.Red
cst['LightBG'].colors.prompt = C.Blue
cst['LightBG'].colors.breakpoint_enabled = C.LightRed
cst['LightBG'].colors.breakpoint_disabled = C.Red
cst['Neutral'].colors.prompt = C.Blue
cst['Neutral'].colors.breakpoint_enabled = C.LightRed
cst['Neutral'].colors.breakpoint_disabled = C.Red
# Add a python parser so we can syntax highlight source while
# debugging.
self.parser = PyColorize.Parser(style=color_scheme)
self.set_colors(color_scheme)
# Set the prompt - the default prompt is '(Pdb)'
self.prompt = prompt
self.skip_hidden = True
self.report_skipped = True
# list of predicates we use to skip frames
self._predicates = self.default_predicates
#
def set_colors(self, scheme):
"""Shorthand access to the color table scheme selector method."""
self.color_scheme_table.set_active_scheme(scheme)
self.parser.style = scheme
def set_trace(self, frame=None):
if frame is None:
frame = sys._getframe().f_back
self.initial_frame = frame
return super().set_trace(frame)
def _hidden_predicate(self, frame):
"""
Given a frame return whether it it should be hidden or not by IPython.
"""
if self._predicates["readonly"]:
fname = frame.f_code.co_filename
# we need to check for file existence and interactively define
# function would otherwise appear as RO.
if os.path.isfile(fname) and not os.access(fname, os.W_OK):
return True
if self._predicates["tbhide"]:
if frame in (self.curframe, getattr(self, "initial_frame", None)):
return False
frame_locals = self._get_frame_locals(frame)
if "__tracebackhide__" not in frame_locals:
return False
return frame_locals["__tracebackhide__"]
return False
def hidden_frames(self, stack):
"""
Given an index in the stack return whether it should be skipped.
This is used in up/down and where to skip frames.
"""
# The f_locals dictionary is updated from the actual frame
# locals whenever the .f_locals accessor is called, so we
# avoid calling it here to preserve self.curframe_locals.
# Furthermore, there is no good reason to hide the current frame.
ip_hide = [self._hidden_predicate(s[0]) for s in stack]
ip_start = [i for i, s in enumerate(ip_hide) if s == "__ipython_bottom__"]
if ip_start and self._predicates["ipython_internal"]:
ip_hide = [h if i > ip_start[0] else True for (i, h) in enumerate(ip_hide)]
return ip_hide
def interaction(self, frame, traceback):
try:
OldPdb.interaction(self, frame, traceback)
except KeyboardInterrupt:
self.stdout.write("\n" + self.shell.get_exception_only())
def precmd(self, line):
"""Perform useful escapes on the command before it is executed."""
if line.endswith("??"):
line = "pinfo2 " + line[:-2]
elif line.endswith("?"):
line = "pinfo " + line[:-1]
line = super().precmd(line)
return line
def new_do_frame(self, arg):
OldPdb.do_frame(self, arg)
def new_do_quit(self, arg):
if hasattr(self, 'old_all_completions'):
self.shell.Completer.all_completions = self.old_all_completions
return OldPdb.do_quit(self, arg)
do_q = do_quit = decorate_fn_with_doc(new_do_quit, OldPdb.do_quit)
def new_do_restart(self, arg):
"""Restart command. In the context of ipython this is exactly the same
thing as 'quit'."""
self.msg("Restart doesn't make sense here. Using 'quit' instead.")
return self.do_quit(arg)
def print_stack_trace(self, context=None):
Colors = self.color_scheme_table.active_colors
ColorsNormal = Colors.Normal
if context is None:
context = self.context
try:
context = int(context)
if context <= 0:
raise ValueError("Context must be a positive integer")
except (TypeError, ValueError) as e:
raise ValueError("Context must be a positive integer") from e
try:
skipped = 0
for hidden, frame_lineno in zip(self.hidden_frames(self.stack), self.stack):
if hidden and self.skip_hidden:
skipped += 1
continue
if skipped:
print(
f"{Colors.excName} [... skipping {skipped} hidden frame(s)]{ColorsNormal}\n"
)
skipped = 0
self.print_stack_entry(frame_lineno, context=context)
if skipped:
print(
f"{Colors.excName} [... skipping {skipped} hidden frame(s)]{ColorsNormal}\n"
)
except KeyboardInterrupt:
pass
def print_stack_entry(self, frame_lineno, prompt_prefix='\n-> ',
context=None):
if context is None:
context = self.context
try:
context = int(context)
if context <= 0:
raise ValueError("Context must be a positive integer")
except (TypeError, ValueError) as e:
raise ValueError("Context must be a positive integer") from e
print(self.format_stack_entry(frame_lineno, '', context), file=self.stdout)
# vds: >>
frame, lineno = frame_lineno
filename = frame.f_code.co_filename
self.shell.hooks.synchronize_with_editor(filename, lineno, 0)
# vds: <<
def _get_frame_locals(self, frame):
""" "
Accessing f_local of current frame reset the namespace, so we want to avoid
that or the following can happen
ipdb> foo
"old"
ipdb> foo = "new"
ipdb> foo
"new"
ipdb> where
ipdb> foo
"old"
So if frame is self.current_frame we instead return self.curframe_locals
"""
if frame is self.curframe:
return self.curframe_locals
else:
return frame.f_locals
def format_stack_entry(self, frame_lineno, lprefix=': ', context=None):
if context is None:
context = self.context
try:
context = int(context)
if context <= 0:
print("Context must be a positive integer", file=self.stdout)
except (TypeError, ValueError):
print("Context must be a positive integer", file=self.stdout)
import reprlib
ret = []
Colors = self.color_scheme_table.active_colors
ColorsNormal = Colors.Normal
tpl_link = "%s%%s%s" % (Colors.filenameEm, ColorsNormal)
tpl_call = "%s%%s%s%%s%s" % (Colors.vName, Colors.valEm, ColorsNormal)
tpl_line = "%%s%s%%s %s%%s" % (Colors.lineno, ColorsNormal)
tpl_line_em = "%%s%s%%s %s%%s%s" % (Colors.linenoEm, Colors.line, ColorsNormal)
frame, lineno = frame_lineno
return_value = ''
loc_frame = self._get_frame_locals(frame)
if "__return__" in loc_frame:
rv = loc_frame["__return__"]
# return_value += '->'
return_value += reprlib.repr(rv) + "\n"
ret.append(return_value)
#s = filename + '(' + `lineno` + ')'
filename = self.canonic(frame.f_code.co_filename)
link = tpl_link % py3compat.cast_unicode(filename)
if frame.f_code.co_name:
func = frame.f_code.co_name
else:
func = "<lambda>"
call = ""
if func != "?":
if "__args__" in loc_frame:
args = reprlib.repr(loc_frame["__args__"])
else:
args = '()'
call = tpl_call % (func, args)
# The level info should be generated in the same format pdb uses, to
# avoid breaking the pdbtrack functionality of python-mode in *emacs.
if frame is self.curframe:
ret.append('> ')
else:
ret.append(" ")
ret.append("%s(%s)%s\n" % (link, lineno, call))
start = lineno - 1 - context//2
lines = linecache.getlines(filename)
start = min(start, len(lines) - context)
start = max(start, 0)
lines = lines[start : start + context]
for i, line in enumerate(lines):
show_arrow = start + 1 + i == lineno
linetpl = (frame is self.curframe or show_arrow) and tpl_line_em or tpl_line
ret.append(
self.__format_line(
linetpl, filename, start + 1 + i, line, arrow=show_arrow
)
)
return "".join(ret)
def __format_line(self, tpl_line, filename, lineno, line, arrow=False):
bp_mark = ""
bp_mark_color = ""
new_line, err = self.parser.format2(line, 'str')
if not err:
line = new_line
bp = None
if lineno in self.get_file_breaks(filename):
bps = self.get_breaks(filename, lineno)
bp = bps[-1]
if bp:
Colors = self.color_scheme_table.active_colors
bp_mark = str(bp.number)
bp_mark_color = Colors.breakpoint_enabled
if not bp.enabled:
bp_mark_color = Colors.breakpoint_disabled
numbers_width = 7
if arrow:
# This is the line with the error
pad = numbers_width - len(str(lineno)) - len(bp_mark)
num = '%s%s' % (make_arrow(pad), str(lineno))
else:
num = '%*s' % (numbers_width - len(bp_mark), str(lineno))
return tpl_line % (bp_mark_color + bp_mark, num, line)
def print_list_lines(self, filename, first, last):
"""The printing (as opposed to the parsing part of a 'list'
command."""
try:
Colors = self.color_scheme_table.active_colors
ColorsNormal = Colors.Normal
tpl_line = '%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal)
tpl_line_em = '%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line, ColorsNormal)
src = []
if filename == "<string>" and hasattr(self, "_exec_filename"):
filename = self._exec_filename
for lineno in range(first, last+1):
line = linecache.getline(filename, lineno)
if not line:
break
if lineno == self.curframe.f_lineno:
line = self.__format_line(
tpl_line_em, filename, lineno, line, arrow=True
)
else:
line = self.__format_line(
tpl_line, filename, lineno, line, arrow=False
)
src.append(line)
self.lineno = lineno
print(''.join(src), file=self.stdout)
except KeyboardInterrupt:
pass
def do_skip_predicates(self, args):
"""
Turn on/off individual predicates as to whether a frame should be hidden/skip.
The global option to skip (or not) hidden frames is set with skip_hidden
To change the value of a predicate
skip_predicates key [true|false]
Call without arguments to see the current values.
To permanently change the value of an option add the corresponding
command to your ``~/.pdbrc`` file. If you are programmatically using the
Pdb instance you can also change the ``default_predicates`` class
attribute.
"""
if not args.strip():
print("current predicates:")
for (p, v) in self._predicates.items():
print(" ", p, ":", v)
return
type_value = args.strip().split(" ")
if len(type_value) != 2:
print(
f"Usage: skip_predicates <type> <value>, with <type> one of {set(self._predicates.keys())}"
)
return
type_, value = type_value
if type_ not in self._predicates:
print(f"{type_!r} not in {set(self._predicates.keys())}")
return
if value.lower() not in ("true", "yes", "1", "no", "false", "0"):
print(
f"{value!r} is invalid - use one of ('true', 'yes', '1', 'no', 'false', '0')"
)
return
self._predicates[type_] = value.lower() in ("true", "yes", "1")
if not any(self._predicates.values()):
print(
"Warning, all predicates set to False, skip_hidden may not have any effects."
)
def do_skip_hidden(self, arg):
"""
Change whether or not we should skip frames with the
__tracebackhide__ attribute.
"""
if not arg.strip():
print(
f"skip_hidden = {self.skip_hidden}, use 'yes','no', 'true', or 'false' to change."
)
elif arg.strip().lower() in ("true", "yes"):
self.skip_hidden = True
elif arg.strip().lower() in ("false", "no"):
self.skip_hidden = False
if not any(self._predicates.values()):
print(
"Warning, all predicates set to False, skip_hidden may not have any effects."
)
def do_list(self, arg):
"""Print lines of code from the current stack frame
"""
self.lastcmd = 'list'
last = None
if arg:
try:
x = eval(arg, {}, {})
if type(x) == type(()):
first, last = x
first = int(first)
last = int(last)
if last < first:
# Assume it's a count
last = first + last
else:
first = max(1, int(x) - 5)
except:
print('*** Error in argument:', repr(arg), file=self.stdout)
return
elif self.lineno is None:
first = max(1, self.curframe.f_lineno - 5)
else:
first = self.lineno + 1
if last is None:
last = first + 10
self.print_list_lines(self.curframe.f_code.co_filename, first, last)
# vds: >>
lineno = first
filename = self.curframe.f_code.co_filename
self.shell.hooks.synchronize_with_editor(filename, lineno, 0)
# vds: <<
do_l = do_list
def getsourcelines(self, obj):
lines, lineno = inspect.findsource(obj)
if inspect.isframe(obj) and obj.f_globals is self._get_frame_locals(obj):
# must be a module frame: do not try to cut a block out of it
return lines, 1
elif inspect.ismodule(obj):
return lines, 1
return inspect.getblock(lines[lineno:]), lineno+1
def do_longlist(self, arg):
"""Print lines of code from the current stack frame.
Shows more lines than 'list' does.
"""
self.lastcmd = 'longlist'
try:
lines, lineno = self.getsourcelines(self.curframe)
except OSError as err:
self.error(err)
return
last = lineno + len(lines)
self.print_list_lines(self.curframe.f_code.co_filename, lineno, last)
do_ll = do_longlist
def do_debug(self, arg):
"""debug code
Enter a recursive debugger that steps through the code
argument (which is an arbitrary expression or statement to be
executed in the current environment).
"""
trace_function = sys.gettrace()
sys.settrace(None)
globals = self.curframe.f_globals
locals = self.curframe_locals
p = self.__class__(completekey=self.completekey,
stdin=self.stdin, stdout=self.stdout)
p.use_rawinput = self.use_rawinput
p.prompt = "(%s) " % self.prompt.strip()
self.message("ENTERING RECURSIVE DEBUGGER")
sys.call_tracing(p.run, (arg, globals, locals))
self.message("LEAVING RECURSIVE DEBUGGER")
sys.settrace(trace_function)
self.lastcmd = p.lastcmd
def do_pdef(self, arg):
"""Print the call signature for any callable object.
The debugger interface to %pdef"""
namespaces = [
("Locals", self.curframe_locals),
("Globals", self.curframe.f_globals),
]
self.shell.find_line_magic("pdef")(arg, namespaces=namespaces)
def do_pdoc(self, arg):
"""Print the docstring for an object.
The debugger interface to %pdoc."""
namespaces = [
("Locals", self.curframe_locals),
("Globals", self.curframe.f_globals),
]
self.shell.find_line_magic("pdoc")(arg, namespaces=namespaces)
def do_pfile(self, arg):
"""Print (or run through pager) the file where an object is defined.
The debugger interface to %pfile.
"""
namespaces = [
("Locals", self.curframe_locals),
("Globals", self.curframe.f_globals),
]
self.shell.find_line_magic("pfile")(arg, namespaces=namespaces)
def do_pinfo(self, arg):
"""Provide detailed information about an object.
The debugger interface to %pinfo, i.e., obj?."""
namespaces = [
("Locals", self.curframe_locals),
("Globals", self.curframe.f_globals),
]
self.shell.find_line_magic("pinfo")(arg, namespaces=namespaces)
def do_pinfo2(self, arg):
"""Provide extra detailed information about an object.
The debugger interface to %pinfo2, i.e., obj??."""
namespaces = [
("Locals", self.curframe_locals),
("Globals", self.curframe.f_globals),
]
self.shell.find_line_magic("pinfo2")(arg, namespaces=namespaces)
def do_psource(self, arg):
"""Print (or run through pager) the source code for an object."""
namespaces = [
("Locals", self.curframe_locals),
("Globals", self.curframe.f_globals),
]
self.shell.find_line_magic("psource")(arg, namespaces=namespaces)
def do_where(self, arg):
"""w(here)
Print a stack trace, with the most recent frame at the bottom.
An arrow indicates the "current frame", which determines the
context of most commands. 'bt' is an alias for this command.
Take a number as argument as an (optional) number of context line to
print"""
if arg:
try:
context = int(arg)
except ValueError as err:
self.error(err)
return
self.print_stack_trace(context)
else:
self.print_stack_trace()
do_w = do_where
def break_anywhere(self, frame):
"""
_stop_in_decorator_internals is overly restrictive, as we may still want
to trace function calls, so we need to also update break_anywhere so
that is we don't `stop_here`, because of debugger skip, we may still
stop at any point inside the function
"""
sup = super().break_anywhere(frame)
if sup:
return sup
if self._predicates["debuggerskip"]:
if DEBUGGERSKIP in frame.f_code.co_varnames:
return True
if frame.f_back and self._get_frame_locals(frame.f_back).get(DEBUGGERSKIP):
return True
return False
def _is_in_decorator_internal_and_should_skip(self, frame):
"""
Utility to tell us whether we are in a decorator internal and should stop.
"""
# if we are disabled don't skip
if not self._predicates["debuggerskip"]:
return False
# if frame is tagged, skip by default.
if DEBUGGERSKIP in frame.f_code.co_varnames:
return True
# if one of the parent frame value set to True skip as well.
cframe = frame
while getattr(cframe, "f_back", None):
cframe = cframe.f_back
if self._get_frame_locals(cframe).get(DEBUGGERSKIP):
return True
return False
def stop_here(self, frame):
if self._is_in_decorator_internal_and_should_skip(frame) is True:
return False
hidden = False
if self.skip_hidden:
hidden = self._hidden_predicate(frame)
if hidden:
if self.report_skipped:
Colors = self.color_scheme_table.active_colors
ColorsNormal = Colors.Normal
print(
f"{Colors.excName} [... skipped 1 hidden frame]{ColorsNormal}\n"
)
return super().stop_here(frame)
def do_up(self, arg):
"""u(p) [count]
Move the current frame count (default one) levels up in the
stack trace (to an older frame).
Will skip hidden frames.
"""
# modified version of upstream that skips
# frames with __tracebackhide__
if self.curindex == 0:
self.error("Oldest frame")
return
try:
count = int(arg or 1)
except ValueError:
self.error("Invalid frame count (%s)" % arg)
return
skipped = 0
if count < 0:
_newframe = 0
else:
counter = 0
hidden_frames = self.hidden_frames(self.stack)
for i in range(self.curindex - 1, -1, -1):
if hidden_frames[i] and self.skip_hidden:
skipped += 1
continue
counter += 1
if counter >= count:
break
else:
# if no break occurred.
self.error(
"all frames above hidden, use `skip_hidden False` to get get into those."
)
return
Colors = self.color_scheme_table.active_colors
ColorsNormal = Colors.Normal
_newframe = i
self._select_frame(_newframe)
if skipped:
print(
f"{Colors.excName} [... skipped {skipped} hidden frame(s)]{ColorsNormal}\n"
)
def do_down(self, arg):
"""d(own) [count]
Move the current frame count (default one) levels down in the
stack trace (to a newer frame).
Will skip hidden frames.
"""
if self.curindex + 1 == len(self.stack):
self.error("Newest frame")
return
try:
count = int(arg or 1)
except ValueError:
self.error("Invalid frame count (%s)" % arg)
return
if count < 0:
_newframe = len(self.stack) - 1
else:
counter = 0
skipped = 0
hidden_frames = self.hidden_frames(self.stack)
for i in range(self.curindex + 1, len(self.stack)):
if hidden_frames[i] and self.skip_hidden:
skipped += 1
continue
counter += 1
if counter >= count:
break
else:
self.error(
"all frames below hidden, use `skip_hidden False` to get get into those."
)
return
Colors = self.color_scheme_table.active_colors
ColorsNormal = Colors.Normal
if skipped:
print(
f"{Colors.excName} [... skipped {skipped} hidden frame(s)]{ColorsNormal}\n"
)
_newframe = i
self._select_frame(_newframe)
do_d = do_down
do_u = do_up
def do_context(self, context):
"""context number_of_lines
Set the number of lines of source code to show when displaying
stacktrace information.
"""
try:
new_context = int(context)
if new_context <= 0:
raise ValueError()
self.context = new_context
except ValueError:
self.error("The 'context' command requires a positive integer argument.")
class InterruptiblePdb(Pdb):
"""Version of debugger where KeyboardInterrupt exits the debugger altogether."""
def cmdloop(self, intro=None):
"""Wrap cmdloop() such that KeyboardInterrupt stops the debugger."""
try:
return OldPdb.cmdloop(self, intro=intro)
except KeyboardInterrupt:
self.stop_here = lambda frame: False
self.do_quit("")
sys.settrace(None)
self.quitting = False
raise
def _cmdloop(self):
while True:
try:
# keyboard interrupts allow for an easy way to cancel
# the current command, so allow them during interactive input
self.allow_kbdint = True
self.cmdloop()
self.allow_kbdint = False
break
except KeyboardInterrupt:
self.message('--KeyboardInterrupt--')
raise
def set_trace(frame=None):
"""
Start debugging from `frame`.
If frame is not specified, debugging starts from caller's frame.
"""
Pdb().set_trace(frame or sys._getframe().f_back)

@ -0,0 +1,391 @@
# -*- coding: utf-8 -*-
"""Top-level display functions for displaying object in different formats."""
# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.
from binascii import b2a_hex
import os
import sys
import warnings
__all__ = ['display', 'clear_output', 'publish_display_data', 'update_display', 'DisplayHandle']
#-----------------------------------------------------------------------------
# utility functions
#-----------------------------------------------------------------------------
def _merge(d1, d2):
"""Like update, but merges sub-dicts instead of clobbering at the top level.
Updates d1 in-place
"""
if not isinstance(d2, dict) or not isinstance(d1, dict):
return d2
for key, value in d2.items():
d1[key] = _merge(d1.get(key), value)
return d1
#-----------------------------------------------------------------------------
# Main functions
#-----------------------------------------------------------------------------
class _Sentinel:
def __repr__(self):
return "<deprecated>"
_sentinel = _Sentinel()
# use * to indicate transient is keyword-only
def publish_display_data(
data, metadata=None, source=_sentinel, *, transient=None, **kwargs
):
"""Publish data and metadata to all frontends.
See the ``display_data`` message in the messaging documentation for
more details about this message type.
Keys of data and metadata can be any mime-type.
Parameters
----------
data : dict
A dictionary having keys that are valid MIME types (like
'text/plain' or 'image/svg+xml') and values that are the data for
that MIME type. The data itself must be a JSON'able data
structure. Minimally all data should have the 'text/plain' data,
which can be displayed by all frontends. If more than the plain
text is given, it is up to the frontend to decide which
representation to use.
metadata : dict
A dictionary for metadata related to the data. This can contain
arbitrary key, value pairs that frontends can use to interpret
the data. mime-type keys matching those in data can be used
to specify metadata about particular representations.
source : str, deprecated
Unused.
transient : dict, keyword-only
A dictionary of transient data, such as display_id.
"""
from IPython.core.interactiveshell import InteractiveShell
if source is not _sentinel:
warnings.warn(
"The `source` parameter emit a deprecation warning since"
" IPython 8.0, it had no effects for a long time and will "
" be removed in future versions.",
DeprecationWarning,
stacklevel=2,
)
display_pub = InteractiveShell.instance().display_pub
# only pass transient if supplied,
# to avoid errors with older ipykernel.
# TODO: We could check for ipykernel version and provide a detailed upgrade message.
if transient:
kwargs['transient'] = transient
display_pub.publish(
data=data,
metadata=metadata,
**kwargs
)
def _new_id():
"""Generate a new random text id with urandom"""
return b2a_hex(os.urandom(16)).decode('ascii')
def display(
*objs,
include=None,
exclude=None,
metadata=None,
transient=None,
display_id=None,
raw=False,
clear=False,
**kwargs
):
"""Display a Python object in all frontends.
By default all representations will be computed and sent to the frontends.
Frontends can decide which representation is used and how.
In terminal IPython this will be similar to using :func:`print`, for use in richer
frontends see Jupyter notebook examples with rich display logic.
Parameters
----------
*objs : object
The Python objects to display.
raw : bool, optional
Are the objects to be displayed already mimetype-keyed dicts of raw display data,
or Python objects that need to be formatted before display? [default: False]
include : list, tuple or set, optional
A list of format type strings (MIME types) to include in the
format data dict. If this is set *only* the format types included
in this list will be computed.
exclude : list, tuple or set, optional
A list of format type strings (MIME types) to exclude in the format
data dict. If this is set all format types will be computed,
except for those included in this argument.
metadata : dict, optional
A dictionary of metadata to associate with the output.
mime-type keys in this dictionary will be associated with the individual
representation formats, if they exist.
transient : dict, optional
A dictionary of transient data to associate with the output.
Data in this dict should not be persisted to files (e.g. notebooks).
display_id : str, bool optional
Set an id for the display.
This id can be used for updating this display area later via update_display.
If given as `True`, generate a new `display_id`
clear : bool, optional
Should the output area be cleared before displaying anything? If True,
this will wait for additional output before clearing. [default: False]
**kwargs : additional keyword-args, optional
Additional keyword-arguments are passed through to the display publisher.
Returns
-------
handle: DisplayHandle
Returns a handle on updatable displays for use with :func:`update_display`,
if `display_id` is given. Returns :any:`None` if no `display_id` is given
(default).
Examples
--------
>>> class Json(object):
... def __init__(self, json):
... self.json = json
... def _repr_pretty_(self, pp, cycle):
... import json
... pp.text(json.dumps(self.json, indent=2))
... def __repr__(self):
... return str(self.json)
...
>>> d = Json({1:2, 3: {4:5}})
>>> print(d)
{1: 2, 3: {4: 5}}
>>> display(d)
{
"1": 2,
"3": {
"4": 5
}
}
>>> def int_formatter(integer, pp, cycle):
... pp.text('I'*integer)
>>> plain = get_ipython().display_formatter.formatters['text/plain']
>>> plain.for_type(int, int_formatter)
<function _repr_pprint at 0x...>
>>> display(7-5)
II
>>> del plain.type_printers[int]
>>> display(7-5)
2
See Also
--------
:func:`update_display`
Notes
-----
In Python, objects can declare their textual representation using the
`__repr__` method. IPython expands on this idea and allows objects to declare
other, rich representations including:
- HTML
- JSON
- PNG
- JPEG
- SVG
- LaTeX
A single object can declare some or all of these representations; all are
handled by IPython's display system.
The main idea of the first approach is that you have to implement special
display methods when you define your class, one for each representation you
want to use. Here is a list of the names of the special methods and the
values they must return:
- `_repr_html_`: return raw HTML as a string, or a tuple (see below).
- `_repr_json_`: return a JSONable dict, or a tuple (see below).
- `_repr_jpeg_`: return raw JPEG data, or a tuple (see below).
- `_repr_png_`: return raw PNG data, or a tuple (see below).
- `_repr_svg_`: return raw SVG data as a string, or a tuple (see below).
- `_repr_latex_`: return LaTeX commands in a string surrounded by "$",
or a tuple (see below).
- `_repr_mimebundle_`: return a full mimebundle containing the mapping
from all mimetypes to data.
Use this for any mime-type not listed above.
The above functions may also return the object's metadata alonside the
data. If the metadata is available, the functions will return a tuple
containing the data and metadata, in that order. If there is no metadata
available, then the functions will return the data only.
When you are directly writing your own classes, you can adapt them for
display in IPython by following the above approach. But in practice, you
often need to work with existing classes that you can't easily modify.
You can refer to the documentation on integrating with the display system in
order to register custom formatters for already existing types
(:ref:`integrating_rich_display`).
.. versionadded:: 5.4 display available without import
.. versionadded:: 6.1 display available without import
Since IPython 5.4 and 6.1 :func:`display` is automatically made available to
the user without import. If you are using display in a document that might
be used in a pure python context or with older version of IPython, use the
following import at the top of your file::
from IPython.display import display
"""
from IPython.core.interactiveshell import InteractiveShell
if not InteractiveShell.initialized():
# Directly print objects.
print(*objs)
return
if transient is None:
transient = {}
if metadata is None:
metadata={}
if display_id:
if display_id is True:
display_id = _new_id()
transient['display_id'] = display_id
if kwargs.get('update') and 'display_id' not in transient:
raise TypeError('display_id required for update_display')
if transient:
kwargs['transient'] = transient
if not objs and display_id:
# if given no objects, but still a request for a display_id,
# we assume the user wants to insert an empty output that
# can be updated later
objs = [{}]
raw = True
if not raw:
format = InteractiveShell.instance().display_formatter.format
if clear:
clear_output(wait=True)
for obj in objs:
if raw:
publish_display_data(data=obj, metadata=metadata, **kwargs)
else:
format_dict, md_dict = format(obj, include=include, exclude=exclude)
if not format_dict:
# nothing to display (e.g. _ipython_display_ took over)
continue
if metadata:
# kwarg-specified metadata gets precedence
_merge(md_dict, metadata)
publish_display_data(data=format_dict, metadata=md_dict, **kwargs)
if display_id:
return DisplayHandle(display_id)
# use * for keyword-only display_id arg
def update_display(obj, *, display_id, **kwargs):
"""Update an existing display by id
Parameters
----------
obj
The object with which to update the display
display_id : keyword-only
The id of the display to update
See Also
--------
:func:`display`
"""
kwargs['update'] = True
display(obj, display_id=display_id, **kwargs)
class DisplayHandle(object):
"""A handle on an updatable display
Call `.update(obj)` to display a new object.
Call `.display(obj`) to add a new instance of this display,
and update existing instances.
See Also
--------
:func:`display`, :func:`update_display`
"""
def __init__(self, display_id=None):
if display_id is None:
display_id = _new_id()
self.display_id = display_id
def __repr__(self):
return "<%s display_id=%s>" % (self.__class__.__name__, self.display_id)
def display(self, obj, **kwargs):
"""Make a new display with my id, updating existing instances.
Parameters
----------
obj
object to display
**kwargs
additional keyword arguments passed to display
"""
display(obj, display_id=self.display_id, **kwargs)
def update(self, obj, **kwargs):
"""Update existing displays with my id
Parameters
----------
obj
object to display
**kwargs
additional keyword arguments passed to update_display
"""
update_display(obj, display_id=self.display_id, **kwargs)
def clear_output(wait=False):
"""Clear the output of the current cell receiving output.
Parameters
----------
wait : bool [default: false]
Wait to clear the output until new output is available to replace it."""
from IPython.core.interactiveshell import InteractiveShell
if InteractiveShell.initialized():
InteractiveShell.instance().display_pub.clear_output(wait)
else:
print('\033[2K\r', end='')
sys.stdout.flush()
print('\033[2K\r', end='')
sys.stderr.flush()

@ -0,0 +1,70 @@
# encoding: utf-8
"""
A context manager for handling sys.displayhook.
Authors:
* Robert Kern
* Brian Granger
"""
#-----------------------------------------------------------------------------
# Copyright (C) 2008-2011 The IPython Development Team
#
# Distributed under the terms of the BSD License. The full license is in
# the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------
#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
import sys
from traitlets.config.configurable import Configurable
from traitlets import Any
#-----------------------------------------------------------------------------
# Classes and functions
#-----------------------------------------------------------------------------
class DisplayTrap(Configurable):
"""Object to manage sys.displayhook.
This came from IPython.core.kernel.display_hook, but is simplified
(no callbacks or formatters) until more of the core is refactored.
"""
hook = Any()
def __init__(self, hook=None):
super(DisplayTrap, self).__init__(hook=hook, config=None)
self.old_hook = None
# We define this to track if a single BuiltinTrap is nested.
# Only turn off the trap when the outermost call to __exit__ is made.
self._nested_level = 0
def __enter__(self):
if self._nested_level == 0:
self.set()
self._nested_level += 1
return self
def __exit__(self, type, value, traceback):
if self._nested_level == 1:
self.unset()
self._nested_level -= 1
# Returning False will cause exceptions to propagate
return False
def set(self):
"""Set the hook."""
if sys.displayhook is not self.hook:
self.old_hook = sys.displayhook
sys.displayhook = self.hook
def unset(self):
"""Unset the hook."""
sys.displayhook = self.old_hook

@ -0,0 +1,325 @@
# -*- coding: utf-8 -*-
"""Displayhook for IPython.
This defines a callable class that IPython uses for `sys.displayhook`.
"""
# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.
import builtins as builtin_mod
import sys
import io as _io
import tokenize
from traitlets.config.configurable import Configurable
from traitlets import Instance, Float
from warnings import warn
# TODO: Move the various attributes (cache_size, [others now moved]). Some
# of these are also attributes of InteractiveShell. They should be on ONE object
# only and the other objects should ask that one object for their values.
class DisplayHook(Configurable):
"""The custom IPython displayhook to replace sys.displayhook.
This class does many things, but the basic idea is that it is a callable
that gets called anytime user code returns a value.
"""
shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
allow_none=True)
exec_result = Instance('IPython.core.interactiveshell.ExecutionResult',
allow_none=True)
cull_fraction = Float(0.2)
def __init__(self, shell=None, cache_size=1000, **kwargs):
super(DisplayHook, self).__init__(shell=shell, **kwargs)
cache_size_min = 3
if cache_size <= 0:
self.do_full_cache = 0
cache_size = 0
elif cache_size < cache_size_min:
self.do_full_cache = 0
cache_size = 0
warn('caching was disabled (min value for cache size is %s).' %
cache_size_min,stacklevel=3)
else:
self.do_full_cache = 1
self.cache_size = cache_size
# we need a reference to the user-level namespace
self.shell = shell
self._,self.__,self.___ = '','',''
# these are deliberately global:
to_user_ns = {'_':self._,'__':self.__,'___':self.___}
self.shell.user_ns.update(to_user_ns)
@property
def prompt_count(self):
return self.shell.execution_count
#-------------------------------------------------------------------------
# Methods used in __call__. Override these methods to modify the behavior
# of the displayhook.
#-------------------------------------------------------------------------
def check_for_underscore(self):
"""Check if the user has set the '_' variable by hand."""
# If something injected a '_' variable in __builtin__, delete
# ipython's automatic one so we don't clobber that. gettext() in
# particular uses _, so we need to stay away from it.
if '_' in builtin_mod.__dict__:
try:
user_value = self.shell.user_ns['_']
if user_value is not self._:
return
del self.shell.user_ns['_']
except KeyError:
pass
def quiet(self):
"""Should we silence the display hook because of ';'?"""
# do not print output if input ends in ';'
try:
cell = self.shell.history_manager.input_hist_parsed[-1]
except IndexError:
# some uses of ipshellembed may fail here
return False
sio = _io.StringIO(cell)
tokens = list(tokenize.generate_tokens(sio.readline))
for token in reversed(tokens):
if token[0] in (tokenize.ENDMARKER, tokenize.NL, tokenize.NEWLINE, tokenize.COMMENT):
continue
if (token[0] == tokenize.OP) and (token[1] == ';'):
return True
else:
return False
def start_displayhook(self):
"""Start the displayhook, initializing resources."""
pass
def write_output_prompt(self):
"""Write the output prompt.
The default implementation simply writes the prompt to
``sys.stdout``.
"""
# Use write, not print which adds an extra space.
sys.stdout.write(self.shell.separate_out)
outprompt = 'Out[{}]: '.format(self.shell.execution_count)
if self.do_full_cache:
sys.stdout.write(outprompt)
def compute_format_data(self, result):
"""Compute format data of the object to be displayed.
The format data is a generalization of the :func:`repr` of an object.
In the default implementation the format data is a :class:`dict` of
key value pair where the keys are valid MIME types and the values
are JSON'able data structure containing the raw data for that MIME
type. It is up to frontends to determine pick a MIME to to use and
display that data in an appropriate manner.
This method only computes the format data for the object and should
NOT actually print or write that to a stream.
Parameters
----------
result : object
The Python object passed to the display hook, whose format will be
computed.
Returns
-------
(format_dict, md_dict) : dict
format_dict is a :class:`dict` whose keys are valid MIME types and values are
JSON'able raw data for that MIME type. It is recommended that
all return values of this should always include the "text/plain"
MIME type representation of the object.
md_dict is a :class:`dict` with the same MIME type keys
of metadata associated with each output.
"""
return self.shell.display_formatter.format(result)
# This can be set to True by the write_output_prompt method in a subclass
prompt_end_newline = False
def write_format_data(self, format_dict, md_dict=None) -> None:
"""Write the format data dict to the frontend.
This default version of this method simply writes the plain text
representation of the object to ``sys.stdout``. Subclasses should
override this method to send the entire `format_dict` to the
frontends.
Parameters
----------
format_dict : dict
The format dict for the object passed to `sys.displayhook`.
md_dict : dict (optional)
The metadata dict to be associated with the display data.
"""
if 'text/plain' not in format_dict:
# nothing to do
return
# We want to print because we want to always make sure we have a
# newline, even if all the prompt separators are ''. This is the
# standard IPython behavior.
result_repr = format_dict['text/plain']
if '\n' in result_repr:
# So that multi-line strings line up with the left column of
# the screen, instead of having the output prompt mess up
# their first line.
# We use the prompt template instead of the expanded prompt
# because the expansion may add ANSI escapes that will interfere
# with our ability to determine whether or not we should add
# a newline.
if not self.prompt_end_newline:
# But avoid extraneous empty lines.
result_repr = '\n' + result_repr
try:
print(result_repr)
except UnicodeEncodeError:
# If a character is not supported by the terminal encoding replace
# it with its \u or \x representation
print(result_repr.encode(sys.stdout.encoding,'backslashreplace').decode(sys.stdout.encoding))
def update_user_ns(self, result):
"""Update user_ns with various things like _, __, _1, etc."""
# Avoid recursive reference when displaying _oh/Out
if self.cache_size and result is not self.shell.user_ns['_oh']:
if len(self.shell.user_ns['_oh']) >= self.cache_size and self.do_full_cache:
self.cull_cache()
# Don't overwrite '_' and friends if '_' is in __builtin__
# (otherwise we cause buggy behavior for things like gettext). and
# do not overwrite _, __ or ___ if one of these has been assigned
# by the user.
update_unders = True
for unders in ['_'*i for i in range(1,4)]:
if not unders in self.shell.user_ns:
continue
if getattr(self, unders) is not self.shell.user_ns.get(unders):
update_unders = False
self.___ = self.__
self.__ = self._
self._ = result
if ('_' not in builtin_mod.__dict__) and (update_unders):
self.shell.push({'_':self._,
'__':self.__,
'___':self.___}, interactive=False)
# hackish access to top-level namespace to create _1,_2... dynamically
to_main = {}
if self.do_full_cache:
new_result = '_%s' % self.prompt_count
to_main[new_result] = result
self.shell.push(to_main, interactive=False)
self.shell.user_ns['_oh'][self.prompt_count] = result
def fill_exec_result(self, result):
if self.exec_result is not None:
self.exec_result.result = result
def log_output(self, format_dict):
"""Log the output."""
if 'text/plain' not in format_dict:
# nothing to do
return
if self.shell.logger.log_output:
self.shell.logger.log_write(format_dict['text/plain'], 'output')
self.shell.history_manager.output_hist_reprs[self.prompt_count] = \
format_dict['text/plain']
def finish_displayhook(self):
"""Finish up all displayhook activities."""
sys.stdout.write(self.shell.separate_out2)
sys.stdout.flush()
def __call__(self, result=None):
"""Printing with history cache management.
This is invoked every time the interpreter needs to print, and is
activated by setting the variable sys.displayhook to it.
"""
self.check_for_underscore()
if result is not None and not self.quiet():
self.start_displayhook()
self.write_output_prompt()
format_dict, md_dict = self.compute_format_data(result)
self.update_user_ns(result)
self.fill_exec_result(result)
if format_dict:
self.write_format_data(format_dict, md_dict)
self.log_output(format_dict)
self.finish_displayhook()
def cull_cache(self):
"""Output cache is full, cull the oldest entries"""
oh = self.shell.user_ns.get('_oh', {})
sz = len(oh)
cull_count = max(int(sz * self.cull_fraction), 2)
warn('Output cache limit (currently {sz} entries) hit.\n'
'Flushing oldest {cull_count} entries.'.format(sz=sz, cull_count=cull_count))
for i, n in enumerate(sorted(oh)):
if i >= cull_count:
break
self.shell.user_ns.pop('_%i' % n, None)
oh.pop(n, None)
def flush(self):
if not self.do_full_cache:
raise ValueError("You shouldn't have reached the cache flush "
"if full caching is not enabled!")
# delete auto-generated vars from global namespace
for n in range(1,self.prompt_count + 1):
key = '_'+repr(n)
try:
del self.shell.user_ns[key]
except: pass
# In some embedded circumstances, the user_ns doesn't have the
# '_oh' key set up.
oh = self.shell.user_ns.get('_oh', None)
if oh is not None:
oh.clear()
# Release our own references to objects:
self._, self.__, self.___ = '', '', ''
if '_' not in builtin_mod.__dict__:
self.shell.user_ns.update({'_':self._,'__':self.__,'___':self.___})
import gc
# TODO: Is this really needed?
# IronPython blocks here forever
if sys.platform != "cli":
gc.collect()
class CapturingDisplayHook(object):
def __init__(self, shell, outputs=None):
self.shell = shell
if outputs is None:
outputs = []
self.outputs = outputs
def __call__(self, result=None):
if result is None:
return
format_dict, md_dict = self.shell.display_formatter.format(result)
self.outputs.append({ 'data': format_dict, 'metadata': md_dict })

@ -0,0 +1,138 @@
"""An interface for publishing rich data to frontends.
There are two components of the display system:
* Display formatters, which take a Python object and compute the
representation of the object in various formats (text, HTML, SVG, etc.).
* The display publisher that is used to send the representation data to the
various frontends.
This module defines the logic display publishing. The display publisher uses
the ``display_data`` message type that is defined in the IPython messaging
spec.
"""
# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.
import sys
from traitlets.config.configurable import Configurable
from traitlets import List
# This used to be defined here - it is imported for backwards compatibility
from .display_functions import publish_display_data
#-----------------------------------------------------------------------------
# Main payload class
#-----------------------------------------------------------------------------
class DisplayPublisher(Configurable):
"""A traited class that publishes display data to frontends.
Instances of this class are created by the main IPython object and should
be accessed there.
"""
def __init__(self, shell=None, *args, **kwargs):
self.shell = shell
super().__init__(*args, **kwargs)
def _validate_data(self, data, metadata=None):
"""Validate the display data.
Parameters
----------
data : dict
The formata data dictionary.
metadata : dict
Any metadata for the data.
"""
if not isinstance(data, dict):
raise TypeError('data must be a dict, got: %r' % data)
if metadata is not None:
if not isinstance(metadata, dict):
raise TypeError('metadata must be a dict, got: %r' % data)
# use * to indicate transient, update are keyword-only
def publish(self, data, metadata=None, source=None, *, transient=None, update=False, **kwargs) -> None:
"""Publish data and metadata to all frontends.
See the ``display_data`` message in the messaging documentation for
more details about this message type.
The following MIME types are currently implemented:
* text/plain
* text/html
* text/markdown
* text/latex
* application/json
* application/javascript
* image/png
* image/jpeg
* image/svg+xml
Parameters
----------
data : dict
A dictionary having keys that are valid MIME types (like
'text/plain' or 'image/svg+xml') and values that are the data for
that MIME type. The data itself must be a JSON'able data
structure. Minimally all data should have the 'text/plain' data,
which can be displayed by all frontends. If more than the plain
text is given, it is up to the frontend to decide which
representation to use.
metadata : dict
A dictionary for metadata related to the data. This can contain
arbitrary key, value pairs that frontends can use to interpret
the data. Metadata specific to each mime-type can be specified
in the metadata dict with the same mime-type keys as
the data itself.
source : str, deprecated
Unused.
transient : dict, keyword-only
A dictionary for transient data.
Data in this dictionary should not be persisted as part of saving this output.
Examples include 'display_id'.
update : bool, keyword-only, default: False
If True, only update existing outputs with the same display_id,
rather than creating a new output.
"""
handlers = {}
if self.shell is not None:
handlers = getattr(self.shell, 'mime_renderers', {})
for mime, handler in handlers.items():
if mime in data:
handler(data[mime], metadata.get(mime, None))
return
if 'text/plain' in data:
print(data['text/plain'])
def clear_output(self, wait=False):
"""Clear the output of the cell receiving output."""
print('\033[2K\r', end='')
sys.stdout.flush()
print('\033[2K\r', end='')
sys.stderr.flush()
class CapturingDisplayPublisher(DisplayPublisher):
"""A DisplayPublisher that stores"""
outputs = List()
def publish(self, data, metadata=None, source=None, *, transient=None, update=False):
self.outputs.append({'data':data, 'metadata':metadata,
'transient':transient, 'update':update})
def clear_output(self, wait=False):
super(CapturingDisplayPublisher, self).clear_output(wait)
# empty the list, *do not* reassign a new list
self.outputs.clear()

@ -0,0 +1,60 @@
# encoding: utf-8
"""
Global exception classes for IPython.core.
Authors:
* Brian Granger
* Fernando Perez
* Min Ragan-Kelley
Notes
-----
"""
#-----------------------------------------------------------------------------
# Copyright (C) 2008 The IPython Development Team
#
# Distributed under the terms of the BSD License. The full license is in
# the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------
#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
#-----------------------------------------------------------------------------
# Exception classes
#-----------------------------------------------------------------------------
class IPythonCoreError(Exception):
pass
class TryNext(IPythonCoreError):
"""Try next hook exception.
Raise this in your hook function to indicate that the next hook handler
should be used to handle the operation.
"""
class UsageError(IPythonCoreError):
"""Error in magic function arguments, etc.
Something that probably won't warrant a full traceback, but should
nevertheless interrupt a macro / batch file.
"""
class StdinNotImplementedError(IPythonCoreError, NotImplementedError):
"""raw_input was requested in a context where it is not supported
For use in IPython kernels, where only some frontends may support
stdin requests.
"""
class InputRejected(Exception):
"""Input rejected by ast transformer.
Raise this in your NodeTransformer to indicate that InteractiveShell should
not execute the supplied input.
"""

@ -0,0 +1,161 @@
"""Infrastructure for registering and firing callbacks on application events.
Unlike :mod:`IPython.core.hooks`, which lets end users set single functions to
be called at specific times, or a collection of alternative methods to try,
callbacks are designed to be used by extension authors. A number of callbacks
can be registered for the same event without needing to be aware of one another.
The functions defined in this module are no-ops indicating the names of available
events and the arguments which will be passed to them.
.. note::
This API is experimental in IPython 2.0, and may be revised in future versions.
"""
from backcall import callback_prototype
class EventManager(object):
"""Manage a collection of events and a sequence of callbacks for each.
This is attached to :class:`~IPython.core.interactiveshell.InteractiveShell`
instances as an ``events`` attribute.
.. note::
This API is experimental in IPython 2.0, and may be revised in future versions.
"""
def __init__(self, shell, available_events):
"""Initialise the :class:`CallbackManager`.
Parameters
----------
shell
The :class:`~IPython.core.interactiveshell.InteractiveShell` instance
available_events
An iterable of names for callback events.
"""
self.shell = shell
self.callbacks = {n:[] for n in available_events}
def register(self, event, function):
"""Register a new event callback.
Parameters
----------
event : str
The event for which to register this callback.
function : callable
A function to be called on the given event. It should take the same
parameters as the appropriate callback prototype.
Raises
------
TypeError
If ``function`` is not callable.
KeyError
If ``event`` is not one of the known events.
"""
if not callable(function):
raise TypeError('Need a callable, got %r' % function)
callback_proto = available_events.get(event)
if function not in self.callbacks[event]:
self.callbacks[event].append(callback_proto.adapt(function))
def unregister(self, event, function):
"""Remove a callback from the given event."""
if function in self.callbacks[event]:
return self.callbacks[event].remove(function)
# Remove callback in case ``function`` was adapted by `backcall`.
for callback in self.callbacks[event]:
try:
if callback.__wrapped__ is function:
return self.callbacks[event].remove(callback)
except AttributeError:
pass
raise ValueError('Function {!r} is not registered as a {} callback'.format(function, event))
def trigger(self, event, *args, **kwargs):
"""Call callbacks for ``event``.
Any additional arguments are passed to all callbacks registered for this
event. Exceptions raised by callbacks are caught, and a message printed.
"""
for func in self.callbacks[event][:]:
try:
func(*args, **kwargs)
except (Exception, KeyboardInterrupt):
print("Error in callback {} (for {}):".format(func, event))
self.shell.showtraceback()
# event_name -> prototype mapping
available_events = {}
def _define_event(callback_function):
callback_proto = callback_prototype(callback_function)
available_events[callback_function.__name__] = callback_proto
return callback_proto
# ------------------------------------------------------------------------------
# Callback prototypes
#
# No-op functions which describe the names of available events and the
# signatures of callbacks for those events.
# ------------------------------------------------------------------------------
@_define_event
def pre_execute():
"""Fires before code is executed in response to user/frontend action.
This includes comm and widget messages and silent execution, as well as user
code cells.
"""
pass
@_define_event
def pre_run_cell(info):
"""Fires before user-entered code runs.
Parameters
----------
info : :class:`~IPython.core.interactiveshell.ExecutionInfo`
An object containing information used for the code execution.
"""
pass
@_define_event
def post_execute():
"""Fires after code is executed in response to user/frontend action.
This includes comm and widget messages and silent execution, as well as user
code cells.
"""
pass
@_define_event
def post_run_cell(result):
"""Fires after user-entered code runs.
Parameters
----------
result : :class:`~IPython.core.interactiveshell.ExecutionResult`
The object which will be returned as the execution result.
"""
pass
@_define_event
def shell_initialized(ip):
"""Fires after initialisation of :class:`~IPython.core.interactiveshell.InteractiveShell`.
This is before extensions and startup scripts are loaded, so it can only be
set by subclassing.
Parameters
----------
ip : :class:`~IPython.core.interactiveshell.InteractiveShell`
The newly initialised shell.
"""
pass

@ -0,0 +1,166 @@
# -*- coding: utf-8 -*-
"""
Color schemes for exception handling code in IPython.
"""
import os
import warnings
#*****************************************************************************
# Copyright (C) 2005-2006 Fernando Perez <fperez@colorado.edu>
#
# Distributed under the terms of the BSD License. The full license is in
# the file COPYING, distributed as part of this software.
#*****************************************************************************
from IPython.utils.coloransi import ColorSchemeTable, TermColors, ColorScheme
def exception_colors():
"""Return a color table with fields for exception reporting.
The table is an instance of ColorSchemeTable with schemes added for
'Neutral', 'Linux', 'LightBG' and 'NoColor' and fields for exception handling filled
in.
Examples:
>>> ec = exception_colors()
>>> ec.active_scheme_name
''
>>> print(ec.active_colors)
None
Now we activate a color scheme:
>>> ec.set_active_scheme('NoColor')
>>> ec.active_scheme_name
'NoColor'
>>> sorted(ec.active_colors.keys())
['Normal', 'caret', 'em', 'excName', 'filename', 'filenameEm', 'line',
'lineno', 'linenoEm', 'name', 'nameEm', 'normalEm', 'topline', 'vName',
'val', 'valEm']
"""
ex_colors = ColorSchemeTable()
# Populate it with color schemes
C = TermColors # shorthand and local lookup
ex_colors.add_scheme(ColorScheme(
'NoColor',
# The color to be used for the top line
topline = C.NoColor,
# The colors to be used in the traceback
filename = C.NoColor,
lineno = C.NoColor,
name = C.NoColor,
vName = C.NoColor,
val = C.NoColor,
em = C.NoColor,
# Emphasized colors for the last frame of the traceback
normalEm = C.NoColor,
filenameEm = C.NoColor,
linenoEm = C.NoColor,
nameEm = C.NoColor,
valEm = C.NoColor,
# Colors for printing the exception
excName = C.NoColor,
line = C.NoColor,
caret = C.NoColor,
Normal = C.NoColor
))
# make some schemes as instances so we can copy them for modification easily
ex_colors.add_scheme(ColorScheme(
'Linux',
# The color to be used for the top line
topline = C.LightRed,
# The colors to be used in the traceback
filename = C.Green,
lineno = C.Green,
name = C.Purple,
vName = C.Cyan,
val = C.Green,
em = C.LightCyan,
# Emphasized colors for the last frame of the traceback
normalEm = C.LightCyan,
filenameEm = C.LightGreen,
linenoEm = C.LightGreen,
nameEm = C.LightPurple,
valEm = C.LightBlue,
# Colors for printing the exception
excName = C.LightRed,
line = C.Yellow,
caret = C.White,
Normal = C.Normal
))
# For light backgrounds, swap dark/light colors
ex_colors.add_scheme(ColorScheme(
'LightBG',
# The color to be used for the top line
topline = C.Red,
# The colors to be used in the traceback
filename = C.LightGreen,
lineno = C.LightGreen,
name = C.LightPurple,
vName = C.Cyan,
val = C.LightGreen,
em = C.Cyan,
# Emphasized colors for the last frame of the traceback
normalEm = C.Cyan,
filenameEm = C.Green,
linenoEm = C.Green,
nameEm = C.Purple,
valEm = C.Blue,
# Colors for printing the exception
excName = C.Red,
#line = C.Brown, # brown often is displayed as yellow
line = C.Red,
caret = C.Normal,
Normal = C.Normal,
))
ex_colors.add_scheme(ColorScheme(
'Neutral',
# The color to be used for the top line
topline = C.Red,
# The colors to be used in the traceback
filename = C.LightGreen,
lineno = C.LightGreen,
name = C.LightPurple,
vName = C.Cyan,
val = C.LightGreen,
em = C.Cyan,
# Emphasized colors for the last frame of the traceback
normalEm = C.Cyan,
filenameEm = C.Green,
linenoEm = C.Green,
nameEm = C.Purple,
valEm = C.Blue,
# Colors for printing the exception
excName = C.Red,
#line = C.Brown, # brown often is displayed as yellow
line = C.Red,
caret = C.Normal,
Normal = C.Normal,
))
# Hack: the 'neutral' colours are not very visible on a dark background on
# Windows. Since Windows command prompts have a dark background by default, and
# relatively few users are likely to alter that, we will use the 'Linux' colours,
# designed for a dark background, as the default on Windows.
if os.name == "nt":
ex_colors.add_scheme(ex_colors['Linux'].copy('Neutral'))
return ex_colors

@ -0,0 +1,167 @@
# encoding: utf-8
"""A class for managing IPython extensions."""
# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.
import os
import os.path
import sys
from importlib import import_module, reload
from traitlets.config.configurable import Configurable
from IPython.utils.path import ensure_dir_exists, compress_user
from IPython.utils.decorators import undoc
from traitlets import Instance
#-----------------------------------------------------------------------------
# Main class
#-----------------------------------------------------------------------------
BUILTINS_EXTS = {"storemagic": False, "autoreload": False}
class ExtensionManager(Configurable):
"""A class to manage IPython extensions.
An IPython extension is an importable Python module that has
a function with the signature::
def load_ipython_extension(ipython):
# Do things with ipython
This function is called after your extension is imported and the
currently active :class:`InteractiveShell` instance is passed as
the only argument. You can do anything you want with IPython at
that point, including defining new magic and aliases, adding new
components, etc.
You can also optionally define an :func:`unload_ipython_extension(ipython)`
function, which will be called if the user unloads or reloads the extension.
The extension manager will only call :func:`load_ipython_extension` again
if the extension is reloaded.
You can put your extension modules anywhere you want, as long as
they can be imported by Python's standard import mechanism. However,
to make it easy to write extensions, you can also put your extensions
in ``os.path.join(self.ipython_dir, 'extensions')``. This directory
is added to ``sys.path`` automatically.
"""
shell = Instance('IPython.core.interactiveshell.InteractiveShellABC', allow_none=True)
def __init__(self, shell=None, **kwargs):
super(ExtensionManager, self).__init__(shell=shell, **kwargs)
self.shell.observe(
self._on_ipython_dir_changed, names=('ipython_dir',)
)
self.loaded = set()
@property
def ipython_extension_dir(self):
return os.path.join(self.shell.ipython_dir, u'extensions')
def _on_ipython_dir_changed(self, change):
ensure_dir_exists(self.ipython_extension_dir)
def load_extension(self, module_str: str):
"""Load an IPython extension by its module name.
Returns the string "already loaded" if the extension is already loaded,
"no load function" if the module doesn't have a load_ipython_extension
function, or None if it succeeded.
"""
try:
return self._load_extension(module_str)
except ModuleNotFoundError:
if module_str in BUILTINS_EXTS:
BUILTINS_EXTS[module_str] = True
return self._load_extension("IPython.extensions." + module_str)
raise
def _load_extension(self, module_str: str):
if module_str in self.loaded:
return "already loaded"
from IPython.utils.syspathcontext import prepended_to_syspath
with self.shell.builtin_trap:
if module_str not in sys.modules:
with prepended_to_syspath(self.ipython_extension_dir):
mod = import_module(module_str)
if mod.__file__.startswith(self.ipython_extension_dir):
print(("Loading extensions from {dir} is deprecated. "
"We recommend managing extensions like any "
"other Python packages, in site-packages.").format(
dir=compress_user(self.ipython_extension_dir)))
mod = sys.modules[module_str]
if self._call_load_ipython_extension(mod):
self.loaded.add(module_str)
else:
return "no load function"
def unload_extension(self, module_str: str):
"""Unload an IPython extension by its module name.
This function looks up the extension's name in ``sys.modules`` and
simply calls ``mod.unload_ipython_extension(self)``.
Returns the string "no unload function" if the extension doesn't define
a function to unload itself, "not loaded" if the extension isn't loaded,
otherwise None.
"""
if BUILTINS_EXTS.get(module_str, False) is True:
module_str = "IPython.extensions." + module_str
if module_str not in self.loaded:
return "not loaded"
if module_str in sys.modules:
mod = sys.modules[module_str]
if self._call_unload_ipython_extension(mod):
self.loaded.discard(module_str)
else:
return "no unload function"
def reload_extension(self, module_str: str):
"""Reload an IPython extension by calling reload.
If the module has not been loaded before,
:meth:`InteractiveShell.load_extension` is called. Otherwise
:func:`reload` is called and then the :func:`load_ipython_extension`
function of the module, if it exists is called.
"""
from IPython.utils.syspathcontext import prepended_to_syspath
if BUILTINS_EXTS.get(module_str, False) is True:
module_str = "IPython.extensions." + module_str
if (module_str in self.loaded) and (module_str in sys.modules):
self.unload_extension(module_str)
mod = sys.modules[module_str]
with prepended_to_syspath(self.ipython_extension_dir):
reload(mod)
if self._call_load_ipython_extension(mod):
self.loaded.add(module_str)
else:
self.load_extension(module_str)
def _call_load_ipython_extension(self, mod):
if hasattr(mod, 'load_ipython_extension'):
mod.load_ipython_extension(self.shell)
return True
def _call_unload_ipython_extension(self, mod):
if hasattr(mod, 'unload_ipython_extension'):
mod.unload_ipython_extension(self.shell)
return True
@undoc
def install_extension(self, url, filename=None):
"""
Deprecated.
"""
# Ensure the extension directory exists
raise DeprecationWarning(
'`install_extension` and the `install_ext` magic have been deprecated since IPython 4.0'
'Use pip or other package managers to manage ipython extensions.')

@ -0,0 +1,24 @@
# encoding: utf-8
"""Simple function to call to get the current InteractiveShell instance
"""
#-----------------------------------------------------------------------------
# Copyright (C) 2013 The IPython Development Team
#
# Distributed under the terms of the BSD License. The full license is in
# the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------
#-----------------------------------------------------------------------------
# Classes and functions
#-----------------------------------------------------------------------------
def get_ipython():
"""Get the global InteractiveShell instance.
Returns None if no InteractiveShell instance is registered.
"""
from IPython.core.interactiveshell import InteractiveShell
if InteractiveShell.initialized():
return InteractiveShell.instance()

@ -0,0 +1,938 @@
""" History related magics and functionality """
# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.
import atexit
import datetime
from pathlib import Path
import re
import sqlite3
import threading
from traitlets.config.configurable import LoggingConfigurable
from decorator import decorator
from IPython.utils.decorators import undoc
from IPython.paths import locate_profile
from traitlets import (
Any,
Bool,
Dict,
Instance,
Integer,
List,
Unicode,
Union,
TraitError,
default,
observe,
)
#-----------------------------------------------------------------------------
# Classes and functions
#-----------------------------------------------------------------------------
@undoc
class DummyDB(object):
"""Dummy DB that will act as a black hole for history.
Only used in the absence of sqlite"""
def execute(*args, **kwargs):
return []
def commit(self, *args, **kwargs):
pass
def __enter__(self, *args, **kwargs):
pass
def __exit__(self, *args, **kwargs):
pass
@decorator
def only_when_enabled(f, self, *a, **kw):
"""Decorator: return an empty list in the absence of sqlite."""
if not self.enabled:
return []
else:
return f(self, *a, **kw)
# use 16kB as threshold for whether a corrupt history db should be saved
# that should be at least 100 entries or so
_SAVE_DB_SIZE = 16384
@decorator
def catch_corrupt_db(f, self, *a, **kw):
"""A decorator which wraps HistoryAccessor method calls to catch errors from
a corrupt SQLite database, move the old database out of the way, and create
a new one.
We avoid clobbering larger databases because this may be triggered due to filesystem issues,
not just a corrupt file.
"""
try:
return f(self, *a, **kw)
except (sqlite3.DatabaseError, sqlite3.OperationalError) as e:
self._corrupt_db_counter += 1
self.log.error("Failed to open SQLite history %s (%s).", self.hist_file, e)
if self.hist_file != ':memory:':
if self._corrupt_db_counter > self._corrupt_db_limit:
self.hist_file = ':memory:'
self.log.error("Failed to load history too many times, history will not be saved.")
elif self.hist_file.is_file():
# move the file out of the way
base = str(self.hist_file.parent / self.hist_file.stem)
ext = self.hist_file.suffix
size = self.hist_file.stat().st_size
if size >= _SAVE_DB_SIZE:
# if there's significant content, avoid clobbering
now = datetime.datetime.now().isoformat().replace(':', '.')
newpath = base + '-corrupt-' + now + ext
# don't clobber previous corrupt backups
for i in range(100):
if not Path(newpath).exists():
break
else:
newpath = base + '-corrupt-' + now + (u'-%i' % i) + ext
else:
# not much content, possibly empty; don't worry about clobbering
# maybe we should just delete it?
newpath = base + '-corrupt' + ext
self.hist_file.rename(newpath)
self.log.error("History file was moved to %s and a new file created.", newpath)
self.init_db()
return []
else:
# Failed with :memory:, something serious is wrong
raise
class HistoryAccessorBase(LoggingConfigurable):
"""An abstract class for History Accessors """
def get_tail(self, n=10, raw=True, output=False, include_latest=False):
raise NotImplementedError
def search(self, pattern="*", raw=True, search_raw=True,
output=False, n=None, unique=False):
raise NotImplementedError
def get_range(self, session, start=1, stop=None, raw=True,output=False):
raise NotImplementedError
def get_range_by_str(self, rangestr, raw=True, output=False):
raise NotImplementedError
class HistoryAccessor(HistoryAccessorBase):
"""Access the history database without adding to it.
This is intended for use by standalone history tools. IPython shells use
HistoryManager, below, which is a subclass of this."""
# counter for init_db retries, so we don't keep trying over and over
_corrupt_db_counter = 0
# after two failures, fallback on :memory:
_corrupt_db_limit = 2
# String holding the path to the history file
hist_file = Union(
[Instance(Path), Unicode()],
help="""Path to file to use for SQLite history database.
By default, IPython will put the history database in the IPython
profile directory. If you would rather share one history among
profiles, you can set this value in each, so that they are consistent.
Due to an issue with fcntl, SQLite is known to misbehave on some NFS
mounts. If you see IPython hanging, try setting this to something on a
local disk, e.g::
ipython --HistoryManager.hist_file=/tmp/ipython_hist.sqlite
you can also use the specific value `:memory:` (including the colon
at both end but not the back ticks), to avoid creating an history file.
""",
).tag(config=True)
enabled = Bool(True,
help="""enable the SQLite history
set enabled=False to disable the SQLite history,
in which case there will be no stored history, no SQLite connection,
and no background saving thread. This may be necessary in some
threaded environments where IPython is embedded.
""",
).tag(config=True)
connection_options = Dict(
help="""Options for configuring the SQLite connection
These options are passed as keyword args to sqlite3.connect
when establishing database connections.
"""
).tag(config=True)
# The SQLite database
db = Any()
@observe('db')
def _db_changed(self, change):
"""validate the db, since it can be an Instance of two different types"""
new = change['new']
connection_types = (DummyDB, sqlite3.Connection)
if not isinstance(new, connection_types):
msg = "%s.db must be sqlite3 Connection or DummyDB, not %r" % \
(self.__class__.__name__, new)
raise TraitError(msg)
def __init__(self, profile="default", hist_file="", **traits):
"""Create a new history accessor.
Parameters
----------
profile : str
The name of the profile from which to open history.
hist_file : str
Path to an SQLite history database stored by IPython. If specified,
hist_file overrides profile.
config : :class:`~traitlets.config.loader.Config`
Config object. hist_file can also be set through this.
"""
# We need a pointer back to the shell for various tasks.
super(HistoryAccessor, self).__init__(**traits)
# defer setting hist_file from kwarg until after init,
# otherwise the default kwarg value would clobber any value
# set by config
if hist_file:
self.hist_file = hist_file
try:
self.hist_file
except TraitError:
# No one has set the hist_file, yet.
self.hist_file = self._get_hist_file_name(profile)
self.init_db()
def _get_hist_file_name(self, profile='default'):
"""Find the history file for the given profile name.
This is overridden by the HistoryManager subclass, to use the shell's
active profile.
Parameters
----------
profile : str
The name of a profile which has a history file.
"""
return Path(locate_profile(profile)) / "history.sqlite"
@catch_corrupt_db
def init_db(self):
"""Connect to the database, and create tables if necessary."""
if not self.enabled:
self.db = DummyDB()
return
# use detect_types so that timestamps return datetime objects
kwargs = dict(detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
kwargs.update(self.connection_options)
self.db = sqlite3.connect(str(self.hist_file), **kwargs)
with self.db:
self.db.execute(
"""CREATE TABLE IF NOT EXISTS sessions (session integer
primary key autoincrement, start timestamp,
end timestamp, num_cmds integer, remark text)"""
)
self.db.execute(
"""CREATE TABLE IF NOT EXISTS history
(session integer, line integer, source text, source_raw text,
PRIMARY KEY (session, line))"""
)
# Output history is optional, but ensure the table's there so it can be
# enabled later.
self.db.execute(
"""CREATE TABLE IF NOT EXISTS output_history
(session integer, line integer, output text,
PRIMARY KEY (session, line))"""
)
# success! reset corrupt db count
self._corrupt_db_counter = 0
def writeout_cache(self):
"""Overridden by HistoryManager to dump the cache before certain
database lookups."""
pass
## -------------------------------
## Methods for retrieving history:
## -------------------------------
def _run_sql(self, sql, params, raw=True, output=False, latest=False):
"""Prepares and runs an SQL query for the history database.
Parameters
----------
sql : str
Any filtering expressions to go after SELECT ... FROM ...
params : tuple
Parameters passed to the SQL query (to replace "?")
raw, output : bool
See :meth:`get_range`
latest : bool
Select rows with max (session, line)
Returns
-------
Tuples as :meth:`get_range`
"""
toget = 'source_raw' if raw else 'source'
sqlfrom = "history"
if output:
sqlfrom = "history LEFT JOIN output_history USING (session, line)"
toget = "history.%s, output_history.output" % toget
if latest:
toget += ", MAX(session * 128 * 1024 + line)"
this_querry = "SELECT session, line, %s FROM %s " % (toget, sqlfrom) + sql
cur = self.db.execute(this_querry, params)
if latest:
cur = (row[:-1] for row in cur)
if output: # Regroup into 3-tuples, and parse JSON
return ((ses, lin, (inp, out)) for ses, lin, inp, out in cur)
return cur
@only_when_enabled
@catch_corrupt_db
def get_session_info(self, session):
"""Get info about a session.
Parameters
----------
session : int
Session number to retrieve.
Returns
-------
session_id : int
Session ID number
start : datetime
Timestamp for the start of the session.
end : datetime
Timestamp for the end of the session, or None if IPython crashed.
num_cmds : int
Number of commands run, or None if IPython crashed.
remark : unicode
A manually set description.
"""
query = "SELECT * from sessions where session == ?"
return self.db.execute(query, (session,)).fetchone()
@catch_corrupt_db
def get_last_session_id(self):
"""Get the last session ID currently in the database.
Within IPython, this should be the same as the value stored in
:attr:`HistoryManager.session_number`.
"""
for record in self.get_tail(n=1, include_latest=True):
return record[0]
@catch_corrupt_db
def get_tail(self, n=10, raw=True, output=False, include_latest=False):
"""Get the last n lines from the history database.
Most recent entry last.
Completion will be reordered so that that the last ones are when
possible from current session.
Parameters
----------
n : int
The number of lines to get
raw, output : bool
See :meth:`get_range`
include_latest : bool
If False (default), n+1 lines are fetched, and the latest one
is discarded. This is intended to be used where the function
is called by a user command, which it should not return.
Returns
-------
Tuples as :meth:`get_range`
"""
self.writeout_cache()
if not include_latest:
n += 1
# cursor/line/entry
this_cur = list(
self._run_sql(
"WHERE session == ? ORDER BY line DESC LIMIT ? ",
(self.session_number, n),
raw=raw,
output=output,
)
)
other_cur = list(
self._run_sql(
"WHERE session != ? ORDER BY session DESC, line DESC LIMIT ?",
(self.session_number, n),
raw=raw,
output=output,
)
)
everything = this_cur + other_cur
everything = everything[:n]
if not include_latest:
return list(everything)[:0:-1]
return list(everything)[::-1]
@catch_corrupt_db
def search(self, pattern="*", raw=True, search_raw=True,
output=False, n=None, unique=False):
"""Search the database using unix glob-style matching (wildcards
* and ?).
Parameters
----------
pattern : str
The wildcarded pattern to match when searching
search_raw : bool
If True, search the raw input, otherwise, the parsed input
raw, output : bool
See :meth:`get_range`
n : None or int
If an integer is given, it defines the limit of
returned entries.
unique : bool
When it is true, return only unique entries.
Returns
-------
Tuples as :meth:`get_range`
"""
tosearch = "source_raw" if search_raw else "source"
if output:
tosearch = "history." + tosearch
self.writeout_cache()
sqlform = "WHERE %s GLOB ?" % tosearch
params = (pattern,)
if unique:
sqlform += ' GROUP BY {0}'.format(tosearch)
if n is not None:
sqlform += " ORDER BY session DESC, line DESC LIMIT ?"
params += (n,)
elif unique:
sqlform += " ORDER BY session, line"
cur = self._run_sql(sqlform, params, raw=raw, output=output, latest=unique)
if n is not None:
return reversed(list(cur))
return cur
@catch_corrupt_db
def get_range(self, session, start=1, stop=None, raw=True,output=False):
"""Retrieve input by session.
Parameters
----------
session : int
Session number to retrieve.
start : int
First line to retrieve.
stop : int
End of line range (excluded from output itself). If None, retrieve
to the end of the session.
raw : bool
If True, return untranslated input
output : bool
If True, attempt to include output. This will be 'real' Python
objects for the current session, or text reprs from previous
sessions if db_log_output was enabled at the time. Where no output
is found, None is used.
Returns
-------
entries
An iterator over the desired lines. Each line is a 3-tuple, either
(session, line, input) if output is False, or
(session, line, (input, output)) if output is True.
"""
if stop:
lineclause = "line >= ? AND line < ?"
params = (session, start, stop)
else:
lineclause = "line>=?"
params = (session, start)
return self._run_sql("WHERE session==? AND %s" % lineclause,
params, raw=raw, output=output)
def get_range_by_str(self, rangestr, raw=True, output=False):
"""Get lines of history from a string of ranges, as used by magic
commands %hist, %save, %macro, etc.
Parameters
----------
rangestr : str
A string specifying ranges, e.g. "5 ~2/1-4". If empty string is used,
this will return everything from current session's history.
See the documentation of :func:`%history` for the full details.
raw, output : bool
As :meth:`get_range`
Returns
-------
Tuples as :meth:`get_range`
"""
for sess, s, e in extract_hist_ranges(rangestr):
for line in self.get_range(sess, s, e, raw=raw, output=output):
yield line
class HistoryManager(HistoryAccessor):
"""A class to organize all history-related functionality in one place.
"""
# Public interface
# An instance of the IPython shell we are attached to
shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
allow_none=True)
# Lists to hold processed and raw history. These start with a blank entry
# so that we can index them starting from 1
input_hist_parsed = List([""])
input_hist_raw = List([""])
# A list of directories visited during session
dir_hist = List()
@default('dir_hist')
def _dir_hist_default(self):
try:
return [Path.cwd()]
except OSError:
return []
# A dict of output history, keyed with ints from the shell's
# execution count.
output_hist = Dict()
# The text/plain repr of outputs.
output_hist_reprs = Dict()
# The number of the current session in the history database
session_number = Integer()
db_log_output = Bool(False,
help="Should the history database include output? (default: no)"
).tag(config=True)
db_cache_size = Integer(0,
help="Write to database every x commands (higher values save disk access & power).\n"
"Values of 1 or less effectively disable caching."
).tag(config=True)
# The input and output caches
db_input_cache = List()
db_output_cache = List()
# History saving in separate thread
save_thread = Instance('IPython.core.history.HistorySavingThread',
allow_none=True)
save_flag = Instance(threading.Event, allow_none=True)
# Private interface
# Variables used to store the three last inputs from the user. On each new
# history update, we populate the user's namespace with these, shifted as
# necessary.
_i00 = Unicode(u'')
_i = Unicode(u'')
_ii = Unicode(u'')
_iii = Unicode(u'')
# A regex matching all forms of the exit command, so that we don't store
# them in the history (it's annoying to rewind the first entry and land on
# an exit call).
_exit_re = re.compile(r"(exit|quit)(\s*\(.*\))?$")
def __init__(self, shell=None, config=None, **traits):
"""Create a new history manager associated with a shell instance.
"""
# We need a pointer back to the shell for various tasks.
super(HistoryManager, self).__init__(shell=shell, config=config,
**traits)
self.save_flag = threading.Event()
self.db_input_cache_lock = threading.Lock()
self.db_output_cache_lock = threading.Lock()
try:
self.new_session()
except sqlite3.OperationalError:
self.log.error("Failed to create history session in %s. History will not be saved.",
self.hist_file, exc_info=True)
self.hist_file = ':memory:'
if self.enabled and self.hist_file != ':memory:':
self.save_thread = HistorySavingThread(self)
self.save_thread.start()
def _get_hist_file_name(self, profile=None):
"""Get default history file name based on the Shell's profile.
The profile parameter is ignored, but must exist for compatibility with
the parent class."""
profile_dir = self.shell.profile_dir.location
return Path(profile_dir) / "history.sqlite"
@only_when_enabled
def new_session(self, conn=None):
"""Get a new session number."""
if conn is None:
conn = self.db
with conn:
cur = conn.execute("""INSERT INTO sessions VALUES (NULL, ?, NULL,
NULL, "") """, (datetime.datetime.now(),))
self.session_number = cur.lastrowid
def end_session(self):
"""Close the database session, filling in the end time and line count."""
self.writeout_cache()
with self.db:
self.db.execute("""UPDATE sessions SET end=?, num_cmds=? WHERE
session==?""", (datetime.datetime.now(),
len(self.input_hist_parsed)-1, self.session_number))
self.session_number = 0
def name_session(self, name):
"""Give the current session a name in the history database."""
with self.db:
self.db.execute("UPDATE sessions SET remark=? WHERE session==?",
(name, self.session_number))
def reset(self, new_session=True):
"""Clear the session history, releasing all object references, and
optionally open a new session."""
self.output_hist.clear()
# The directory history can't be completely empty
self.dir_hist[:] = [Path.cwd()]
if new_session:
if self.session_number:
self.end_session()
self.input_hist_parsed[:] = [""]
self.input_hist_raw[:] = [""]
self.new_session()
# ------------------------------
# Methods for retrieving history
# ------------------------------
def get_session_info(self, session=0):
"""Get info about a session.
Parameters
----------
session : int
Session number to retrieve. The current session is 0, and negative
numbers count back from current session, so -1 is the previous session.
Returns
-------
session_id : int
Session ID number
start : datetime
Timestamp for the start of the session.
end : datetime
Timestamp for the end of the session, or None if IPython crashed.
num_cmds : int
Number of commands run, or None if IPython crashed.
remark : unicode
A manually set description.
"""
if session <= 0:
session += self.session_number
return super(HistoryManager, self).get_session_info(session=session)
def _get_range_session(self, start=1, stop=None, raw=True, output=False):
"""Get input and output history from the current session. Called by
get_range, and takes similar parameters."""
input_hist = self.input_hist_raw if raw else self.input_hist_parsed
n = len(input_hist)
if start < 0:
start += n
if not stop or (stop > n):
stop = n
elif stop < 0:
stop += n
for i in range(start, stop):
if output:
line = (input_hist[i], self.output_hist_reprs.get(i))
else:
line = input_hist[i]
yield (0, i, line)
def get_range(self, session=0, start=1, stop=None, raw=True,output=False):
"""Retrieve input by session.
Parameters
----------
session : int
Session number to retrieve. The current session is 0, and negative
numbers count back from current session, so -1 is previous session.
start : int
First line to retrieve.
stop : int
End of line range (excluded from output itself). If None, retrieve
to the end of the session.
raw : bool
If True, return untranslated input
output : bool
If True, attempt to include output. This will be 'real' Python
objects for the current session, or text reprs from previous
sessions if db_log_output was enabled at the time. Where no output
is found, None is used.
Returns
-------
entries
An iterator over the desired lines. Each line is a 3-tuple, either
(session, line, input) if output is False, or
(session, line, (input, output)) if output is True.
"""
if session <= 0:
session += self.session_number
if session==self.session_number: # Current session
return self._get_range_session(start, stop, raw, output)
return super(HistoryManager, self).get_range(session, start, stop, raw,
output)
## ----------------------------
## Methods for storing history:
## ----------------------------
def store_inputs(self, line_num, source, source_raw=None):
"""Store source and raw input in history and create input cache
variables ``_i*``.
Parameters
----------
line_num : int
The prompt number of this input.
source : str
Python input.
source_raw : str, optional
If given, this is the raw input without any IPython transformations
applied to it. If not given, ``source`` is used.
"""
if source_raw is None:
source_raw = source
source = source.rstrip('\n')
source_raw = source_raw.rstrip('\n')
# do not store exit/quit commands
if self._exit_re.match(source_raw.strip()):
return
self.input_hist_parsed.append(source)
self.input_hist_raw.append(source_raw)
with self.db_input_cache_lock:
self.db_input_cache.append((line_num, source, source_raw))
# Trigger to flush cache and write to DB.
if len(self.db_input_cache) >= self.db_cache_size:
self.save_flag.set()
# update the auto _i variables
self._iii = self._ii
self._ii = self._i
self._i = self._i00
self._i00 = source_raw
# hackish access to user namespace to create _i1,_i2... dynamically
new_i = '_i%s' % line_num
to_main = {'_i': self._i,
'_ii': self._ii,
'_iii': self._iii,
new_i : self._i00 }
if self.shell is not None:
self.shell.push(to_main, interactive=False)
def store_output(self, line_num):
"""If database output logging is enabled, this saves all the
outputs from the indicated prompt number to the database. It's
called by run_cell after code has been executed.
Parameters
----------
line_num : int
The line number from which to save outputs
"""
if (not self.db_log_output) or (line_num not in self.output_hist_reprs):
return
output = self.output_hist_reprs[line_num]
with self.db_output_cache_lock:
self.db_output_cache.append((line_num, output))
if self.db_cache_size <= 1:
self.save_flag.set()
def _writeout_input_cache(self, conn):
with conn:
for line in self.db_input_cache:
conn.execute("INSERT INTO history VALUES (?, ?, ?, ?)",
(self.session_number,)+line)
def _writeout_output_cache(self, conn):
with conn:
for line in self.db_output_cache:
conn.execute("INSERT INTO output_history VALUES (?, ?, ?)",
(self.session_number,)+line)
@only_when_enabled
def writeout_cache(self, conn=None):
"""Write any entries in the cache to the database."""
if conn is None:
conn = self.db
with self.db_input_cache_lock:
try:
self._writeout_input_cache(conn)
except sqlite3.IntegrityError:
self.new_session(conn)
print("ERROR! Session/line number was not unique in",
"database. History logging moved to new session",
self.session_number)
try:
# Try writing to the new session. If this fails, don't
# recurse
self._writeout_input_cache(conn)
except sqlite3.IntegrityError:
pass
finally:
self.db_input_cache = []
with self.db_output_cache_lock:
try:
self._writeout_output_cache(conn)
except sqlite3.IntegrityError:
print("!! Session/line number for output was not unique",
"in database. Output will not be stored.")
finally:
self.db_output_cache = []
class HistorySavingThread(threading.Thread):
"""This thread takes care of writing history to the database, so that
the UI isn't held up while that happens.
It waits for the HistoryManager's save_flag to be set, then writes out
the history cache. The main thread is responsible for setting the flag when
the cache size reaches a defined threshold."""
daemon = True
stop_now = False
enabled = True
def __init__(self, history_manager):
super(HistorySavingThread, self).__init__(name="IPythonHistorySavingThread")
self.history_manager = history_manager
self.enabled = history_manager.enabled
atexit.register(self.stop)
@only_when_enabled
def run(self):
# We need a separate db connection per thread:
try:
self.db = sqlite3.connect(
str(self.history_manager.hist_file),
**self.history_manager.connection_options,
)
while True:
self.history_manager.save_flag.wait()
if self.stop_now:
self.db.close()
return
self.history_manager.save_flag.clear()
self.history_manager.writeout_cache(self.db)
except Exception as e:
print(("The history saving thread hit an unexpected error (%s)."
"History will not be written to the database.") % repr(e))
def stop(self):
"""This can be called from the main thread to safely stop this thread.
Note that it does not attempt to write out remaining history before
exiting. That should be done by calling the HistoryManager's
end_session method."""
self.stop_now = True
self.history_manager.save_flag.set()
self.join()
# To match, e.g. ~5/8-~2/3
range_re = re.compile(r"""
((?P<startsess>~?\d+)/)?
(?P<start>\d+)?
((?P<sep>[\-:])
((?P<endsess>~?\d+)/)?
(?P<end>\d+))?
$""", re.VERBOSE)
def extract_hist_ranges(ranges_str):
"""Turn a string of history ranges into 3-tuples of (session, start, stop).
Empty string results in a `[(0, 1, None)]`, i.e. "everything from current
session".
Examples
--------
>>> list(extract_hist_ranges("~8/5-~7/4 2"))
[(-8, 5, None), (-7, 1, 5), (0, 2, 3)]
"""
if ranges_str == "":
yield (0, 1, None) # Everything from current session
return
for range_str in ranges_str.split():
rmatch = range_re.match(range_str)
if not rmatch:
continue
start = rmatch.group("start")
if start:
start = int(start)
end = rmatch.group("end")
# If no end specified, get (a, a + 1)
end = int(end) if end else start + 1
else: # start not specified
if not rmatch.group('startsess'): # no startsess
continue
start = 1
end = None # provide the entire session hist
if rmatch.group("sep") == "-": # 1-3 == 1:4 --> [1, 2, 3]
end += 1
startsess = rmatch.group("startsess") or "0"
endsess = rmatch.group("endsess") or startsess
startsess = int(startsess.replace("~","-"))
endsess = int(endsess.replace("~","-"))
assert endsess >= startsess, "start session must be earlier than end session"
if endsess == startsess:
yield (startsess, start, end)
continue
# Multiple sessions in one range:
yield (startsess, start, None)
for sess in range(startsess+1, endsess):
yield (sess, 1, None)
yield (endsess, 1, end)
def _format_lineno(session, line):
"""Helper function to format line numbers properly."""
if session == 0:
return str(line)
return "%s#%s" % (session, line)

@ -0,0 +1,161 @@
# encoding: utf-8
"""
An application for managing IPython history.
To be invoked as the `ipython history` subcommand.
"""
import sqlite3
from pathlib import Path
from traitlets.config.application import Application
from .application import BaseIPythonApplication
from traitlets import Bool, Int, Dict
from ..utils.io import ask_yes_no
trim_hist_help = """Trim the IPython history database to the last 1000 entries.
This actually copies the last 1000 entries to a new database, and then replaces
the old file with the new. Use the `--keep=` argument to specify a number
other than 1000.
"""
clear_hist_help = """Clear the IPython history database, deleting all entries.
Because this is a destructive operation, IPython will prompt the user if they
really want to do this. Passing a `-f` flag will force clearing without a
prompt.
This is an handy alias to `ipython history trim --keep=0`
"""
class HistoryTrim(BaseIPythonApplication):
description = trim_hist_help
backup = Bool(False,
help="Keep the old history file as history.sqlite.<N>"
).tag(config=True)
keep = Int(1000,
help="Number of recent lines to keep in the database."
).tag(config=True)
flags = Dict(dict(
backup = ({'HistoryTrim' : {'backup' : True}},
backup.help
)
))
aliases=Dict(dict(
keep = 'HistoryTrim.keep'
))
def start(self):
profile_dir = Path(self.profile_dir.location)
hist_file = profile_dir / "history.sqlite"
con = sqlite3.connect(hist_file)
# Grab the recent history from the current database.
inputs = list(con.execute('SELECT session, line, source, source_raw FROM '
'history ORDER BY session DESC, line DESC LIMIT ?', (self.keep+1,)))
if len(inputs) <= self.keep:
print("There are already at most %d entries in the history database." % self.keep)
print("Not doing anything. Use --keep= argument to keep fewer entries")
return
print("Trimming history to the most recent %d entries." % self.keep)
inputs.pop() # Remove the extra element we got to check the length.
inputs.reverse()
if inputs:
first_session = inputs[0][0]
outputs = list(con.execute('SELECT session, line, output FROM '
'output_history WHERE session >= ?', (first_session,)))
sessions = list(con.execute('SELECT session, start, end, num_cmds, remark FROM '
'sessions WHERE session >= ?', (first_session,)))
con.close()
# Create the new history database.
new_hist_file = profile_dir / "history.sqlite.new"
i = 0
while new_hist_file.exists():
# Make sure we don't interfere with an existing file.
i += 1
new_hist_file = profile_dir / ("history.sqlite.new" + str(i))
new_db = sqlite3.connect(new_hist_file)
new_db.execute("""CREATE TABLE IF NOT EXISTS sessions (session integer
primary key autoincrement, start timestamp,
end timestamp, num_cmds integer, remark text)""")
new_db.execute("""CREATE TABLE IF NOT EXISTS history
(session integer, line integer, source text, source_raw text,
PRIMARY KEY (session, line))""")
new_db.execute("""CREATE TABLE IF NOT EXISTS output_history
(session integer, line integer, output text,
PRIMARY KEY (session, line))""")
new_db.commit()
if inputs:
with new_db:
# Add the recent history into the new database.
new_db.executemany('insert into sessions values (?,?,?,?,?)', sessions)
new_db.executemany('insert into history values (?,?,?,?)', inputs)
new_db.executemany('insert into output_history values (?,?,?)', outputs)
new_db.close()
if self.backup:
i = 1
backup_hist_file = profile_dir / ("history.sqlite.old.%d" % i)
while backup_hist_file.exists():
i += 1
backup_hist_file = profile_dir / ("history.sqlite.old.%d" % i)
hist_file.rename(backup_hist_file)
print("Backed up longer history file to", backup_hist_file)
else:
hist_file.unlink()
new_hist_file.rename(hist_file)
class HistoryClear(HistoryTrim):
description = clear_hist_help
keep = Int(0,
help="Number of recent lines to keep in the database.")
force = Bool(False,
help="Don't prompt user for confirmation"
).tag(config=True)
flags = Dict(dict(
force = ({'HistoryClear' : {'force' : True}},
force.help),
f = ({'HistoryTrim' : {'force' : True}},
force.help
)
))
aliases = Dict()
def start(self):
if self.force or ask_yes_no("Really delete all ipython history? ",
default="no", interrupt="no"):
HistoryTrim.start(self)
class HistoryApp(Application):
name = u'ipython-history'
description = "Manage the IPython history database."
subcommands = Dict(dict(
trim = (HistoryTrim, HistoryTrim.description.splitlines()[0]),
clear = (HistoryClear, HistoryClear.description.splitlines()[0]),
))
def start(self):
if self.subapp is None:
print("No subcommand specified. Must specify one of: %s" % \
(self.subcommands.keys()))
print()
self.print_description()
self.print_subcommands()
self.exit(1)
else:
return self.subapp.start()

@ -0,0 +1,171 @@
"""Hooks for IPython.
In Python, it is possible to overwrite any method of any object if you really
want to. But IPython exposes a few 'hooks', methods which are *designed* to
be overwritten by users for customization purposes. This module defines the
default versions of all such hooks, which get used by IPython if not
overridden by the user.
Hooks are simple functions, but they should be declared with ``self`` as their
first argument, because when activated they are registered into IPython as
instance methods. The self argument will be the IPython running instance
itself, so hooks have full access to the entire IPython object.
If you wish to define a new hook and activate it, you can make an :doc:`extension
</config/extensions/index>` or a :ref:`startup script <startup_files>`. For
example, you could use a startup file like this::
import os
def calljed(self,filename, linenum):
"My editor hook calls the jed editor directly."
print "Calling my own editor, jed ..."
if os.system('jed +%d %s' % (linenum,filename)) != 0:
raise TryNext()
def load_ipython_extension(ip):
ip.set_hook('editor', calljed)
"""
#*****************************************************************************
# Copyright (C) 2005 Fernando Perez. <fperez@colorado.edu>
#
# Distributed under the terms of the BSD License. The full license is in
# the file COPYING, distributed as part of this software.
#*****************************************************************************
import os
import subprocess
import sys
from .error import TryNext
# List here all the default hooks. For now it's just the editor functions
# but over time we'll move here all the public API for user-accessible things.
__all__ = [
"editor",
"synchronize_with_editor",
"show_in_pager",
"pre_prompt_hook",
"clipboard_get",
]
deprecated = {'pre_run_code_hook': "a callback for the 'pre_execute' or 'pre_run_cell' event",
'late_startup_hook': "a callback for the 'shell_initialized' event",
'shutdown_hook': "the atexit module",
}
def editor(self, filename, linenum=None, wait=True):
"""Open the default editor at the given filename and linenumber.
This is IPython's default editor hook, you can use it as an example to
write your own modified one. To set your own editor function as the
new editor hook, call ip.set_hook('editor',yourfunc)."""
# IPython configures a default editor at startup by reading $EDITOR from
# the environment, and falling back on vi (unix) or notepad (win32).
editor = self.editor
# marker for at which line to open the file (for existing objects)
if linenum is None or editor=='notepad':
linemark = ''
else:
linemark = '+%d' % int(linenum)
# Enclose in quotes if necessary and legal
if ' ' in editor and os.path.isfile(editor) and editor[0] != '"':
editor = '"%s"' % editor
# Call the actual editor
proc = subprocess.Popen('%s %s %s' % (editor, linemark, filename),
shell=True)
if wait and proc.wait() != 0:
raise TryNext()
def synchronize_with_editor(self, filename, linenum, column):
pass
class CommandChainDispatcher:
""" Dispatch calls to a chain of commands until some func can handle it
Usage: instantiate, execute "add" to add commands (with optional
priority), execute normally via f() calling mechanism.
"""
def __init__(self,commands=None):
if commands is None:
self.chain = []
else:
self.chain = commands
def __call__(self,*args, **kw):
""" Command chain is called just like normal func.
This will call all funcs in chain with the same args as were given to
this function, and return the result of first func that didn't raise
TryNext"""
last_exc = TryNext()
for prio,cmd in self.chain:
#print "prio",prio,"cmd",cmd #dbg
try:
return cmd(*args, **kw)
except TryNext as exc:
last_exc = exc
# if no function will accept it, raise TryNext up to the caller
raise last_exc
def __str__(self):
return str(self.chain)
def add(self, func, priority=0):
""" Add a func to the cmd chain with given priority """
self.chain.append((priority, func))
self.chain.sort(key=lambda x: x[0])
def __iter__(self):
""" Return all objects in chain.
Handy if the objects are not callable.
"""
return iter(self.chain)
def show_in_pager(self, data, start, screen_lines):
""" Run a string through pager """
# raising TryNext here will use the default paging functionality
raise TryNext
def pre_prompt_hook(self):
""" Run before displaying the next prompt
Use this e.g. to display output from asynchronous operations (in order
to not mess up text entry)
"""
return None
def clipboard_get(self):
""" Get text from the clipboard.
"""
from ..lib.clipboard import (
osx_clipboard_get, tkinter_clipboard_get,
win32_clipboard_get
)
if sys.platform == 'win32':
chain = [win32_clipboard_get, tkinter_clipboard_get]
elif sys.platform == 'darwin':
chain = [osx_clipboard_get, tkinter_clipboard_get]
else:
chain = [tkinter_clipboard_get]
dispatcher = CommandChainDispatcher()
for func in chain:
dispatcher.add(func)
text = dispatcher()
return text

@ -0,0 +1,772 @@
"""DEPRECATED: Input handling and transformation machinery.
This module was deprecated in IPython 7.0, in favour of inputtransformer2.
The first class in this module, :class:`InputSplitter`, is designed to tell when
input from a line-oriented frontend is complete and should be executed, and when
the user should be prompted for another line of code instead. The name 'input
splitter' is largely for historical reasons.
A companion, :class:`IPythonInputSplitter`, provides the same functionality but
with full support for the extended IPython syntax (magics, system calls, etc).
The code to actually do these transformations is in :mod:`IPython.core.inputtransformer`.
:class:`IPythonInputSplitter` feeds the raw code to the transformers in order
and stores the results.
For more details, see the class docstrings below.
"""
from warnings import warn
warn('IPython.core.inputsplitter is deprecated since IPython 7 in favor of `IPython.core.inputtransformer2`',
DeprecationWarning)
# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.
import ast
import codeop
import io
import re
import sys
import tokenize
import warnings
from IPython.core.inputtransformer import (leading_indent,
classic_prompt,
ipy_prompt,
cellmagic,
assemble_logical_lines,
help_end,
escaped_commands,
assign_from_magic,
assign_from_system,
assemble_python_lines,
)
# These are available in this module for backwards compatibility.
from IPython.core.inputtransformer import (ESC_SHELL, ESC_SH_CAP, ESC_HELP,
ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,
ESC_QUOTE, ESC_QUOTE2, ESC_PAREN, ESC_SEQUENCES)
#-----------------------------------------------------------------------------
# Utilities
#-----------------------------------------------------------------------------
# FIXME: These are general-purpose utilities that later can be moved to the
# general ward. Kept here for now because we're being very strict about test
# coverage with this code, and this lets us ensure that we keep 100% coverage
# while developing.
# compiled regexps for autoindent management
dedent_re = re.compile('|'.join([
r'^\s+raise(\s.*)?$', # raise statement (+ space + other stuff, maybe)
r'^\s+raise\([^\)]*\).*$', # wacky raise with immediate open paren
r'^\s+return(\s.*)?$', # normal return (+ space + other stuff, maybe)
r'^\s+return\([^\)]*\).*$', # wacky return with immediate open paren
r'^\s+pass\s*$', # pass (optionally followed by trailing spaces)
r'^\s+break\s*$', # break (optionally followed by trailing spaces)
r'^\s+continue\s*$', # continue (optionally followed by trailing spaces)
]))
ini_spaces_re = re.compile(r'^([ \t\r\f\v]+)')
# regexp to match pure comment lines so we don't accidentally insert 'if 1:'
# before pure comments
comment_line_re = re.compile(r'^\s*\#')
def num_ini_spaces(s):
"""Return the number of initial spaces in a string.
Note that tabs are counted as a single space. For now, we do *not* support
mixing of tabs and spaces in the user's input.
Parameters
----------
s : string
Returns
-------
n : int
"""
ini_spaces = ini_spaces_re.match(s)
if ini_spaces:
return ini_spaces.end()
else:
return 0
# Fake token types for partial_tokenize:
INCOMPLETE_STRING = tokenize.N_TOKENS
IN_MULTILINE_STATEMENT = tokenize.N_TOKENS + 1
# The 2 classes below have the same API as TokenInfo, but don't try to look up
# a token type name that they won't find.
class IncompleteString:
type = exact_type = INCOMPLETE_STRING
def __init__(self, s, start, end, line):
self.s = s
self.start = start
self.end = end
self.line = line
class InMultilineStatement:
type = exact_type = IN_MULTILINE_STATEMENT
def __init__(self, pos, line):
self.s = ''
self.start = self.end = pos
self.line = line
def partial_tokens(s):
"""Iterate over tokens from a possibly-incomplete string of code.
This adds two special token types: INCOMPLETE_STRING and
IN_MULTILINE_STATEMENT. These can only occur as the last token yielded, and
represent the two main ways for code to be incomplete.
"""
readline = io.StringIO(s).readline
token = tokenize.TokenInfo(tokenize.NEWLINE, '', (1, 0), (1, 0), '')
try:
for token in tokenize.generate_tokens(readline):
yield token
except tokenize.TokenError as e:
# catch EOF error
lines = s.splitlines(keepends=True)
end = len(lines), len(lines[-1])
if 'multi-line string' in e.args[0]:
l, c = start = token.end
s = lines[l-1][c:] + ''.join(lines[l:])
yield IncompleteString(s, start, end, lines[-1])
elif 'multi-line statement' in e.args[0]:
yield InMultilineStatement(end, lines[-1])
else:
raise
def find_next_indent(code):
"""Find the number of spaces for the next line of indentation"""
tokens = list(partial_tokens(code))
if tokens[-1].type == tokenize.ENDMARKER:
tokens.pop()
if not tokens:
return 0
while (tokens[-1].type in {tokenize.DEDENT, tokenize.NEWLINE, tokenize.COMMENT}):
tokens.pop()
if tokens[-1].type == INCOMPLETE_STRING:
# Inside a multiline string
return 0
# Find the indents used before
prev_indents = [0]
def _add_indent(n):
if n != prev_indents[-1]:
prev_indents.append(n)
tokiter = iter(tokens)
for tok in tokiter:
if tok.type in {tokenize.INDENT, tokenize.DEDENT}:
_add_indent(tok.end[1])
elif (tok.type == tokenize.NL):
try:
_add_indent(next(tokiter).start[1])
except StopIteration:
break
last_indent = prev_indents.pop()
# If we've just opened a multiline statement (e.g. 'a = ['), indent more
if tokens[-1].type == IN_MULTILINE_STATEMENT:
if tokens[-2].exact_type in {tokenize.LPAR, tokenize.LSQB, tokenize.LBRACE}:
return last_indent + 4
return last_indent
if tokens[-1].exact_type == tokenize.COLON:
# Line ends with colon - indent
return last_indent + 4
if last_indent:
# Examine the last line for dedent cues - statements like return or
# raise which normally end a block of code.
last_line_starts = 0
for i, tok in enumerate(tokens):
if tok.type == tokenize.NEWLINE:
last_line_starts = i + 1
last_line_tokens = tokens[last_line_starts:]
names = [t.string for t in last_line_tokens if t.type == tokenize.NAME]
if names and names[0] in {'raise', 'return', 'pass', 'break', 'continue'}:
# Find the most recent indentation less than the current level
for indent in reversed(prev_indents):
if indent < last_indent:
return indent
return last_indent
def last_blank(src):
"""Determine if the input source ends in a blank.
A blank is either a newline or a line consisting of whitespace.
Parameters
----------
src : string
A single or multiline string.
"""
if not src: return False
ll = src.splitlines()[-1]
return (ll == '') or ll.isspace()
last_two_blanks_re = re.compile(r'\n\s*\n\s*$', re.MULTILINE)
last_two_blanks_re2 = re.compile(r'.+\n\s*\n\s+$', re.MULTILINE)
def last_two_blanks(src):
"""Determine if the input source ends in two blanks.
A blank is either a newline or a line consisting of whitespace.
Parameters
----------
src : string
A single or multiline string.
"""
if not src: return False
# The logic here is tricky: I couldn't get a regexp to work and pass all
# the tests, so I took a different approach: split the source by lines,
# grab the last two and prepend '###\n' as a stand-in for whatever was in
# the body before the last two lines. Then, with that structure, it's
# possible to analyze with two regexps. Not the most elegant solution, but
# it works. If anyone tries to change this logic, make sure to validate
# the whole test suite first!
new_src = '\n'.join(['###\n'] + src.splitlines()[-2:])
return (bool(last_two_blanks_re.match(new_src)) or
bool(last_two_blanks_re2.match(new_src)) )
def remove_comments(src):
"""Remove all comments from input source.
Note: comments are NOT recognized inside of strings!
Parameters
----------
src : string
A single or multiline input string.
Returns
-------
String with all Python comments removed.
"""
return re.sub('#.*', '', src)
def get_input_encoding():
"""Return the default standard input encoding.
If sys.stdin has no encoding, 'ascii' is returned."""
# There are strange environments for which sys.stdin.encoding is None. We
# ensure that a valid encoding is returned.
encoding = getattr(sys.stdin, 'encoding', None)
if encoding is None:
encoding = 'ascii'
return encoding
#-----------------------------------------------------------------------------
# Classes and functions for normal Python syntax handling
#-----------------------------------------------------------------------------
class InputSplitter(object):
r"""An object that can accumulate lines of Python source before execution.
This object is designed to be fed python source line-by-line, using
:meth:`push`. It will return on each push whether the currently pushed
code could be executed already. In addition, it provides a method called
:meth:`push_accepts_more` that can be used to query whether more input
can be pushed into a single interactive block.
This is a simple example of how an interactive terminal-based client can use
this tool::
isp = InputSplitter()
while isp.push_accepts_more():
indent = ' '*isp.indent_spaces
prompt = '>>> ' + indent
line = indent + raw_input(prompt)
isp.push(line)
print 'Input source was:\n', isp.source_reset(),
"""
# A cache for storing the current indentation
# The first value stores the most recently processed source input
# The second value is the number of spaces for the current indentation
# If self.source matches the first value, the second value is a valid
# current indentation. Otherwise, the cache is invalid and the indentation
# must be recalculated.
_indent_spaces_cache = None, None
# String, indicating the default input encoding. It is computed by default
# at initialization time via get_input_encoding(), but it can be reset by a
# client with specific knowledge of the encoding.
encoding = ''
# String where the current full source input is stored, properly encoded.
# Reading this attribute is the normal way of querying the currently pushed
# source code, that has been properly encoded.
source = ''
# Code object corresponding to the current source. It is automatically
# synced to the source, so it can be queried at any time to obtain the code
# object; it will be None if the source doesn't compile to valid Python.
code = None
# Private attributes
# List with lines of input accumulated so far
_buffer = None
# Command compiler
_compile = None
# Boolean indicating whether the current block is complete
_is_complete = None
# Boolean indicating whether the current block has an unrecoverable syntax error
_is_invalid = False
def __init__(self):
"""Create a new InputSplitter instance.
"""
self._buffer = []
self._compile = codeop.CommandCompiler()
self.encoding = get_input_encoding()
def reset(self):
"""Reset the input buffer and associated state."""
self._buffer[:] = []
self.source = ''
self.code = None
self._is_complete = False
self._is_invalid = False
def source_reset(self):
"""Return the input source and perform a full reset.
"""
out = self.source
self.reset()
return out
def check_complete(self, source):
"""Return whether a block of code is ready to execute, or should be continued
This is a non-stateful API, and will reset the state of this InputSplitter.
Parameters
----------
source : string
Python input code, which can be multiline.
Returns
-------
status : str
One of 'complete', 'incomplete', or 'invalid' if source is not a
prefix of valid code.
indent_spaces : int or None
The number of spaces by which to indent the next line of code. If
status is not 'incomplete', this is None.
"""
self.reset()
try:
self.push(source)
except SyntaxError:
# Transformers in IPythonInputSplitter can raise SyntaxError,
# which push() will not catch.
return 'invalid', None
else:
if self._is_invalid:
return 'invalid', None
elif self.push_accepts_more():
return 'incomplete', self.get_indent_spaces()
else:
return 'complete', None
finally:
self.reset()
def push(self, lines:str) -> bool:
"""Push one or more lines of input.
This stores the given lines and returns a status code indicating
whether the code forms a complete Python block or not.
Any exceptions generated in compilation are swallowed, but if an
exception was produced, the method returns True.
Parameters
----------
lines : string
One or more lines of Python input.
Returns
-------
is_complete : boolean
True if the current input source (the result of the current input
plus prior inputs) forms a complete Python execution block. Note that
this value is also stored as a private attribute (``_is_complete``), so it
can be queried at any time.
"""
assert isinstance(lines, str)
self._store(lines)
source = self.source
# Before calling _compile(), reset the code object to None so that if an
# exception is raised in compilation, we don't mislead by having
# inconsistent code/source attributes.
self.code, self._is_complete = None, None
self._is_invalid = False
# Honor termination lines properly
if source.endswith('\\\n'):
return False
try:
with warnings.catch_warnings():
warnings.simplefilter('error', SyntaxWarning)
self.code = self._compile(source, symbol="exec")
# Invalid syntax can produce any of a number of different errors from
# inside the compiler, so we have to catch them all. Syntax errors
# immediately produce a 'ready' block, so the invalid Python can be
# sent to the kernel for evaluation with possible ipython
# special-syntax conversion.
except (SyntaxError, OverflowError, ValueError, TypeError,
MemoryError, SyntaxWarning):
self._is_complete = True
self._is_invalid = True
else:
# Compilation didn't produce any exceptions (though it may not have
# given a complete code object)
self._is_complete = self.code is not None
return self._is_complete
def push_accepts_more(self):
"""Return whether a block of interactive input can accept more input.
This method is meant to be used by line-oriented frontends, who need to
guess whether a block is complete or not based solely on prior and
current input lines. The InputSplitter considers it has a complete
interactive block and will not accept more input when either:
* A SyntaxError is raised
* The code is complete and consists of a single line or a single
non-compound statement
* The code is complete and has a blank line at the end
If the current input produces a syntax error, this method immediately
returns False but does *not* raise the syntax error exception, as
typically clients will want to send invalid syntax to an execution
backend which might convert the invalid syntax into valid Python via
one of the dynamic IPython mechanisms.
"""
# With incomplete input, unconditionally accept more
# A syntax error also sets _is_complete to True - see push()
if not self._is_complete:
#print("Not complete") # debug
return True
# The user can make any (complete) input execute by leaving a blank line
last_line = self.source.splitlines()[-1]
if (not last_line) or last_line.isspace():
#print("Blank line") # debug
return False
# If there's just a single line or AST node, and we're flush left, as is
# the case after a simple statement such as 'a=1', we want to execute it
# straight away.
if self.get_indent_spaces() == 0:
if len(self.source.splitlines()) <= 1:
return False
try:
code_ast = ast.parse(u''.join(self._buffer))
except Exception:
#print("Can't parse AST") # debug
return False
else:
if len(code_ast.body) == 1 and \
not hasattr(code_ast.body[0], 'body'):
#print("Simple statement") # debug
return False
# General fallback - accept more code
return True
def get_indent_spaces(self):
sourcefor, n = self._indent_spaces_cache
if sourcefor == self.source:
return n
# self.source always has a trailing newline
n = find_next_indent(self.source[:-1])
self._indent_spaces_cache = (self.source, n)
return n
# Backwards compatibility. I think all code that used .indent_spaces was
# inside IPython, but we can leave this here until IPython 7 in case any
# other modules are using it. -TK, November 2017
indent_spaces = property(get_indent_spaces)
def _store(self, lines, buffer=None, store='source'):
"""Store one or more lines of input.
If input lines are not newline-terminated, a newline is automatically
appended."""
if buffer is None:
buffer = self._buffer
if lines.endswith('\n'):
buffer.append(lines)
else:
buffer.append(lines+'\n')
setattr(self, store, self._set_source(buffer))
def _set_source(self, buffer):
return u''.join(buffer)
class IPythonInputSplitter(InputSplitter):
"""An input splitter that recognizes all of IPython's special syntax."""
# String with raw, untransformed input.
source_raw = ''
# Flag to track when a transformer has stored input that it hasn't given
# back yet.
transformer_accumulating = False
# Flag to track when assemble_python_lines has stored input that it hasn't
# given back yet.
within_python_line = False
# Private attributes
# List with lines of raw input accumulated so far.
_buffer_raw = None
def __init__(self, line_input_checker=True, physical_line_transforms=None,
logical_line_transforms=None, python_line_transforms=None):
super(IPythonInputSplitter, self).__init__()
self._buffer_raw = []
self._validate = True
if physical_line_transforms is not None:
self.physical_line_transforms = physical_line_transforms
else:
self.physical_line_transforms = [
leading_indent(),
classic_prompt(),
ipy_prompt(),
cellmagic(end_on_blank_line=line_input_checker),
]
self.assemble_logical_lines = assemble_logical_lines()
if logical_line_transforms is not None:
self.logical_line_transforms = logical_line_transforms
else:
self.logical_line_transforms = [
help_end(),
escaped_commands(),
assign_from_magic(),
assign_from_system(),
]
self.assemble_python_lines = assemble_python_lines()
if python_line_transforms is not None:
self.python_line_transforms = python_line_transforms
else:
# We don't use any of these at present
self.python_line_transforms = []
@property
def transforms(self):
"Quick access to all transformers."
return self.physical_line_transforms + \
[self.assemble_logical_lines] + self.logical_line_transforms + \
[self.assemble_python_lines] + self.python_line_transforms
@property
def transforms_in_use(self):
"""Transformers, excluding logical line transformers if we're in a
Python line."""
t = self.physical_line_transforms[:]
if not self.within_python_line:
t += [self.assemble_logical_lines] + self.logical_line_transforms
return t + [self.assemble_python_lines] + self.python_line_transforms
def reset(self):
"""Reset the input buffer and associated state."""
super(IPythonInputSplitter, self).reset()
self._buffer_raw[:] = []
self.source_raw = ''
self.transformer_accumulating = False
self.within_python_line = False
for t in self.transforms:
try:
t.reset()
except SyntaxError:
# Nothing that calls reset() expects to handle transformer
# errors
pass
def flush_transformers(self):
def _flush(transform, outs):
"""yield transformed lines
always strings, never None
transform: the current transform
outs: an iterable of previously transformed inputs.
Each may be multiline, which will be passed
one line at a time to transform.
"""
for out in outs:
for line in out.splitlines():
# push one line at a time
tmp = transform.push(line)
if tmp is not None:
yield tmp
# reset the transform
tmp = transform.reset()
if tmp is not None:
yield tmp
out = []
for t in self.transforms_in_use:
out = _flush(t, out)
out = list(out)
if out:
self._store('\n'.join(out))
def raw_reset(self):
"""Return raw input only and perform a full reset.
"""
out = self.source_raw
self.reset()
return out
def source_reset(self):
try:
self.flush_transformers()
return self.source
finally:
self.reset()
def push_accepts_more(self):
if self.transformer_accumulating:
return True
else:
return super(IPythonInputSplitter, self).push_accepts_more()
def transform_cell(self, cell):
"""Process and translate a cell of input.
"""
self.reset()
try:
self.push(cell)
self.flush_transformers()
return self.source
finally:
self.reset()
def push(self, lines:str) -> bool:
"""Push one or more lines of IPython input.
This stores the given lines and returns a status code indicating
whether the code forms a complete Python block or not, after processing
all input lines for special IPython syntax.
Any exceptions generated in compilation are swallowed, but if an
exception was produced, the method returns True.
Parameters
----------
lines : string
One or more lines of Python input.
Returns
-------
is_complete : boolean
True if the current input source (the result of the current input
plus prior inputs) forms a complete Python execution block. Note that
this value is also stored as a private attribute (_is_complete), so it
can be queried at any time.
"""
assert isinstance(lines, str)
# We must ensure all input is pure unicode
# ''.splitlines() --> [], but we need to push the empty line to transformers
lines_list = lines.splitlines()
if not lines_list:
lines_list = ['']
# Store raw source before applying any transformations to it. Note
# that this must be done *after* the reset() call that would otherwise
# flush the buffer.
self._store(lines, self._buffer_raw, 'source_raw')
transformed_lines_list = []
for line in lines_list:
transformed = self._transform_line(line)
if transformed is not None:
transformed_lines_list.append(transformed)
if transformed_lines_list:
transformed_lines = '\n'.join(transformed_lines_list)
return super(IPythonInputSplitter, self).push(transformed_lines)
else:
# Got nothing back from transformers - they must be waiting for
# more input.
return False
def _transform_line(self, line):
"""Push a line of input code through the various transformers.
Returns any output from the transformers, or None if a transformer
is accumulating lines.
Sets self.transformer_accumulating as a side effect.
"""
def _accumulating(dbg):
#print(dbg)
self.transformer_accumulating = True
return None
for transformer in self.physical_line_transforms:
line = transformer.push(line)
if line is None:
return _accumulating(transformer)
if not self.within_python_line:
line = self.assemble_logical_lines.push(line)
if line is None:
return _accumulating('acc logical line')
for transformer in self.logical_line_transforms:
line = transformer.push(line)
if line is None:
return _accumulating(transformer)
line = self.assemble_python_lines.push(line)
if line is None:
self.within_python_line = True
return _accumulating('acc python line')
else:
self.within_python_line = False
for transformer in self.python_line_transforms:
line = transformer.push(line)
if line is None:
return _accumulating(transformer)
#print("transformers clear") #debug
self.transformer_accumulating = False
return line

@ -0,0 +1,536 @@
"""DEPRECATED: Input transformer classes to support IPython special syntax.
This module was deprecated in IPython 7.0, in favour of inputtransformer2.
This includes the machinery to recognise and transform ``%magic`` commands,
``!system`` commands, ``help?`` querying, prompt stripping, and so forth.
"""
import abc
import functools
import re
import tokenize
from tokenize import generate_tokens, untokenize, TokenError
from io import StringIO
from IPython.core.splitinput import LineInfo
#-----------------------------------------------------------------------------
# Globals
#-----------------------------------------------------------------------------
# The escape sequences that define the syntax transformations IPython will
# apply to user input. These can NOT be just changed here: many regular
# expressions and other parts of the code may use their hardcoded values, and
# for all intents and purposes they constitute the 'IPython syntax', so they
# should be considered fixed.
ESC_SHELL = '!' # Send line to underlying system shell
ESC_SH_CAP = '!!' # Send line to system shell and capture output
ESC_HELP = '?' # Find information about object
ESC_HELP2 = '??' # Find extra-detailed information about object
ESC_MAGIC = '%' # Call magic function
ESC_MAGIC2 = '%%' # Call cell-magic function
ESC_QUOTE = ',' # Split args on whitespace, quote each as string and call
ESC_QUOTE2 = ';' # Quote all args as a single string, call
ESC_PAREN = '/' # Call first argument with rest of line as arguments
ESC_SEQUENCES = [ESC_SHELL, ESC_SH_CAP, ESC_HELP ,\
ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,\
ESC_QUOTE, ESC_QUOTE2, ESC_PAREN ]
class InputTransformer(metaclass=abc.ABCMeta):
"""Abstract base class for line-based input transformers."""
@abc.abstractmethod
def push(self, line):
"""Send a line of input to the transformer, returning the transformed
input or None if the transformer is waiting for more input.
Must be overridden by subclasses.
Implementations may raise ``SyntaxError`` if the input is invalid. No
other exceptions may be raised.
"""
pass
@abc.abstractmethod
def reset(self):
"""Return, transformed any lines that the transformer has accumulated,
and reset its internal state.
Must be overridden by subclasses.
"""
pass
@classmethod
def wrap(cls, func):
"""Can be used by subclasses as a decorator, to return a factory that
will allow instantiation with the decorated object.
"""
@functools.wraps(func)
def transformer_factory(**kwargs):
return cls(func, **kwargs)
return transformer_factory
class StatelessInputTransformer(InputTransformer):
"""Wrapper for a stateless input transformer implemented as a function."""
def __init__(self, func):
self.func = func
def __repr__(self):
return "StatelessInputTransformer(func={0!r})".format(self.func)
def push(self, line):
"""Send a line of input to the transformer, returning the
transformed input."""
return self.func(line)
def reset(self):
"""No-op - exists for compatibility."""
pass
class CoroutineInputTransformer(InputTransformer):
"""Wrapper for an input transformer implemented as a coroutine."""
def __init__(self, coro, **kwargs):
# Prime it
self.coro = coro(**kwargs)
next(self.coro)
def __repr__(self):
return "CoroutineInputTransformer(coro={0!r})".format(self.coro)
def push(self, line):
"""Send a line of input to the transformer, returning the
transformed input or None if the transformer is waiting for more
input.
"""
return self.coro.send(line)
def reset(self):
"""Return, transformed any lines that the transformer has
accumulated, and reset its internal state.
"""
return self.coro.send(None)
class TokenInputTransformer(InputTransformer):
"""Wrapper for a token-based input transformer.
func should accept a list of tokens (5-tuples, see tokenize docs), and
return an iterable which can be passed to tokenize.untokenize().
"""
def __init__(self, func):
self.func = func
self.buf = []
self.reset_tokenizer()
def reset_tokenizer(self):
it = iter(self.buf)
self.tokenizer = generate_tokens(it.__next__)
def push(self, line):
self.buf.append(line + '\n')
if all(l.isspace() for l in self.buf):
return self.reset()
tokens = []
stop_at_NL = False
try:
for intok in self.tokenizer:
tokens.append(intok)
t = intok[0]
if t == tokenize.NEWLINE or (stop_at_NL and t == tokenize.NL):
# Stop before we try to pull a line we don't have yet
break
elif t == tokenize.ERRORTOKEN:
stop_at_NL = True
except TokenError:
# Multi-line statement - stop and try again with the next line
self.reset_tokenizer()
return None
return self.output(tokens)
def output(self, tokens):
self.buf.clear()
self.reset_tokenizer()
return untokenize(self.func(tokens)).rstrip('\n')
def reset(self):
l = ''.join(self.buf)
self.buf.clear()
self.reset_tokenizer()
if l:
return l.rstrip('\n')
class assemble_python_lines(TokenInputTransformer):
def __init__(self):
super(assemble_python_lines, self).__init__(None)
def output(self, tokens):
return self.reset()
@CoroutineInputTransformer.wrap
def assemble_logical_lines():
r"""Join lines following explicit line continuations (\)"""
line = ''
while True:
line = (yield line)
if not line or line.isspace():
continue
parts = []
while line is not None:
if line.endswith('\\') and (not has_comment(line)):
parts.append(line[:-1])
line = (yield None) # Get another line
else:
parts.append(line)
break
# Output
line = ''.join(parts)
# Utilities
def _make_help_call(target, esc, lspace):
"""Prepares a pinfo(2)/psearch call from a target name and the escape
(i.e. ? or ??)"""
method = 'pinfo2' if esc == '??' \
else 'psearch' if '*' in target \
else 'pinfo'
arg = " ".join([method, target])
#Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
t_magic_name, _, t_magic_arg_s = arg.partition(' ')
t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
return "%sget_ipython().run_line_magic(%r, %r)" % (
lspace,
t_magic_name,
t_magic_arg_s,
)
# These define the transformations for the different escape characters.
def _tr_system(line_info):
"Translate lines escaped with: !"
cmd = line_info.line.lstrip().lstrip(ESC_SHELL)
return '%sget_ipython().system(%r)' % (line_info.pre, cmd)
def _tr_system2(line_info):
"Translate lines escaped with: !!"
cmd = line_info.line.lstrip()[2:]
return '%sget_ipython().getoutput(%r)' % (line_info.pre, cmd)
def _tr_help(line_info):
"Translate lines escaped with: ?/??"
# A naked help line should just fire the intro help screen
if not line_info.line[1:]:
return 'get_ipython().show_usage()'
return _make_help_call(line_info.ifun, line_info.esc, line_info.pre)
def _tr_magic(line_info):
"Translate lines escaped with: %"
tpl = '%sget_ipython().run_line_magic(%r, %r)'
if line_info.line.startswith(ESC_MAGIC2):
return line_info.line
cmd = ' '.join([line_info.ifun, line_info.the_rest]).strip()
#Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
t_magic_name, _, t_magic_arg_s = cmd.partition(' ')
t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
return tpl % (line_info.pre, t_magic_name, t_magic_arg_s)
def _tr_quote(line_info):
"Translate lines escaped with: ,"
return '%s%s("%s")' % (line_info.pre, line_info.ifun,
'", "'.join(line_info.the_rest.split()) )
def _tr_quote2(line_info):
"Translate lines escaped with: ;"
return '%s%s("%s")' % (line_info.pre, line_info.ifun,
line_info.the_rest)
def _tr_paren(line_info):
"Translate lines escaped with: /"
return '%s%s(%s)' % (line_info.pre, line_info.ifun,
", ".join(line_info.the_rest.split()))
tr = { ESC_SHELL : _tr_system,
ESC_SH_CAP : _tr_system2,
ESC_HELP : _tr_help,
ESC_HELP2 : _tr_help,
ESC_MAGIC : _tr_magic,
ESC_QUOTE : _tr_quote,
ESC_QUOTE2 : _tr_quote2,
ESC_PAREN : _tr_paren }
@StatelessInputTransformer.wrap
def escaped_commands(line):
"""Transform escaped commands - %magic, !system, ?help + various autocalls.
"""
if not line or line.isspace():
return line
lineinf = LineInfo(line)
if lineinf.esc not in tr:
return line
return tr[lineinf.esc](lineinf)
_initial_space_re = re.compile(r'\s*')
_help_end_re = re.compile(r"""(%{0,2}
(?!\d)[\w*]+ # Variable name
(\.(?!\d)[\w*]+)* # .etc.etc
)
(\?\??)$ # ? or ??
""",
re.VERBOSE)
# Extra pseudotokens for multiline strings and data structures
_MULTILINE_STRING = object()
_MULTILINE_STRUCTURE = object()
def _line_tokens(line):
"""Helper for has_comment and ends_in_comment_or_string."""
readline = StringIO(line).readline
toktypes = set()
try:
for t in generate_tokens(readline):
toktypes.add(t[0])
except TokenError as e:
# There are only two cases where a TokenError is raised.
if 'multi-line string' in e.args[0]:
toktypes.add(_MULTILINE_STRING)
else:
toktypes.add(_MULTILINE_STRUCTURE)
return toktypes
def has_comment(src):
"""Indicate whether an input line has (i.e. ends in, or is) a comment.
This uses tokenize, so it can distinguish comments from # inside strings.
Parameters
----------
src : string
A single line input string.
Returns
-------
comment : bool
True if source has a comment.
"""
return (tokenize.COMMENT in _line_tokens(src))
def ends_in_comment_or_string(src):
"""Indicates whether or not an input line ends in a comment or within
a multiline string.
Parameters
----------
src : string
A single line input string.
Returns
-------
comment : bool
True if source ends in a comment or multiline string.
"""
toktypes = _line_tokens(src)
return (tokenize.COMMENT in toktypes) or (_MULTILINE_STRING in toktypes)
@StatelessInputTransformer.wrap
def help_end(line):
"""Translate lines with ?/?? at the end"""
m = _help_end_re.search(line)
if m is None or ends_in_comment_or_string(line):
return line
target = m.group(1)
esc = m.group(3)
lspace = _initial_space_re.match(line).group(0)
return _make_help_call(target, esc, lspace)
@CoroutineInputTransformer.wrap
def cellmagic(end_on_blank_line=False):
"""Captures & transforms cell magics.
After a cell magic is started, this stores up any lines it gets until it is
reset (sent None).
"""
tpl = 'get_ipython().run_cell_magic(%r, %r, %r)'
cellmagic_help_re = re.compile(r'%%\w+\?')
line = ''
while True:
line = (yield line)
# consume leading empty lines
while not line:
line = (yield line)
if not line.startswith(ESC_MAGIC2):
# This isn't a cell magic, idle waiting for reset then start over
while line is not None:
line = (yield line)
continue
if cellmagic_help_re.match(line):
# This case will be handled by help_end
continue
first = line
body = []
line = (yield None)
while (line is not None) and \
((line.strip() != '') or not end_on_blank_line):
body.append(line)
line = (yield None)
# Output
magic_name, _, first = first.partition(' ')
magic_name = magic_name.lstrip(ESC_MAGIC2)
line = tpl % (magic_name, first, u'\n'.join(body))
def _strip_prompts(prompt_re, initial_re=None, turnoff_re=None):
"""Remove matching input prompts from a block of input.
Parameters
----------
prompt_re : regular expression
A regular expression matching any input prompt (including continuation)
initial_re : regular expression, optional
A regular expression matching only the initial prompt, but not continuation.
If no initial expression is given, prompt_re will be used everywhere.
Used mainly for plain Python prompts, where the continuation prompt
``...`` is a valid Python expression in Python 3, so shouldn't be stripped.
Notes
-----
If `initial_re` and `prompt_re differ`,
only `initial_re` will be tested against the first line.
If any prompt is found on the first two lines,
prompts will be stripped from the rest of the block.
"""
if initial_re is None:
initial_re = prompt_re
line = ''
while True:
line = (yield line)
# First line of cell
if line is None:
continue
out, n1 = initial_re.subn('', line, count=1)
if turnoff_re and not n1:
if turnoff_re.match(line):
# We're in e.g. a cell magic; disable this transformer for
# the rest of the cell.
while line is not None:
line = (yield line)
continue
line = (yield out)
if line is None:
continue
# check for any prompt on the second line of the cell,
# because people often copy from just after the first prompt,
# so we might not see it in the first line.
out, n2 = prompt_re.subn('', line, count=1)
line = (yield out)
if n1 or n2:
# Found a prompt in the first two lines - check for it in
# the rest of the cell as well.
while line is not None:
line = (yield prompt_re.sub('', line, count=1))
else:
# Prompts not in input - wait for reset
while line is not None:
line = (yield line)
@CoroutineInputTransformer.wrap
def classic_prompt():
"""Strip the >>>/... prompts of the Python interactive shell."""
# FIXME: non-capturing version (?:...) usable?
prompt_re = re.compile(r'^(>>>|\.\.\.)( |$)')
initial_re = re.compile(r'^>>>( |$)')
# Any %magic/!system is IPython syntax, so we needn't look for >>> prompts
turnoff_re = re.compile(r'^[%!]')
return _strip_prompts(prompt_re, initial_re, turnoff_re)
@CoroutineInputTransformer.wrap
def ipy_prompt():
"""Strip IPython's In [1]:/...: prompts."""
# FIXME: non-capturing version (?:...) usable?
prompt_re = re.compile(r'^(In \[\d+\]: |\s*\.{3,}: ?)')
# Disable prompt stripping inside cell magics
turnoff_re = re.compile(r'^%%')
return _strip_prompts(prompt_re, turnoff_re=turnoff_re)
@CoroutineInputTransformer.wrap
def leading_indent():
"""Remove leading indentation.
If the first line starts with a spaces or tabs, the same whitespace will be
removed from each following line until it is reset.
"""
space_re = re.compile(r'^[ \t]+')
line = ''
while True:
line = (yield line)
if line is None:
continue
m = space_re.match(line)
if m:
space = m.group(0)
while line is not None:
if line.startswith(space):
line = line[len(space):]
line = (yield line)
else:
# No leading spaces - wait for reset
while line is not None:
line = (yield line)
_assign_pat = \
r'''(?P<lhs>(\s*)
([\w\.]+) # Initial identifier
(\s*,\s*
\*?[\w\.]+)* # Further identifiers for unpacking
\s*?,? # Trailing comma
)
\s*=\s*
'''
assign_system_re = re.compile(r'{}!\s*(?P<cmd>.*)'.format(_assign_pat), re.VERBOSE)
assign_system_template = '%s = get_ipython().getoutput(%r)'
@StatelessInputTransformer.wrap
def assign_from_system(line):
"""Transform assignment from system commands (e.g. files = !ls)"""
m = assign_system_re.match(line)
if m is None:
return line
return assign_system_template % m.group('lhs', 'cmd')
assign_magic_re = re.compile(r'{}%\s*(?P<cmd>.*)'.format(_assign_pat), re.VERBOSE)
assign_magic_template = '%s = get_ipython().run_line_magic(%r, %r)'
@StatelessInputTransformer.wrap
def assign_from_magic(line):
"""Transform assignment from magic commands (e.g. a = %who_ls)"""
m = assign_magic_re.match(line)
if m is None:
return line
#Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
m_lhs, m_cmd = m.group('lhs', 'cmd')
t_magic_name, _, t_magic_arg_s = m_cmd.partition(' ')
t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
return assign_magic_template % (m_lhs, t_magic_name, t_magic_arg_s)

@ -0,0 +1,788 @@
"""Input transformer machinery to support IPython special syntax.
This includes the machinery to recognise and transform ``%magic`` commands,
``!system`` commands, ``help?`` querying, prompt stripping, and so forth.
Added: IPython 7.0. Replaces inputsplitter and inputtransformer which were
deprecated in 7.0.
"""
# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.
import ast
import sys
from codeop import CommandCompiler, Compile
import re
import tokenize
from typing import List, Tuple, Optional, Any
import warnings
_indent_re = re.compile(r'^[ \t]+')
def leading_empty_lines(lines):
"""Remove leading empty lines
If the leading lines are empty or contain only whitespace, they will be
removed.
"""
if not lines:
return lines
for i, line in enumerate(lines):
if line and not line.isspace():
return lines[i:]
return lines
def leading_indent(lines):
"""Remove leading indentation.
If the first line starts with a spaces or tabs, the same whitespace will be
removed from each following line in the cell.
"""
if not lines:
return lines
m = _indent_re.match(lines[0])
if not m:
return lines
space = m.group(0)
n = len(space)
return [l[n:] if l.startswith(space) else l
for l in lines]
class PromptStripper:
"""Remove matching input prompts from a block of input.
Parameters
----------
prompt_re : regular expression
A regular expression matching any input prompt (including continuation,
e.g. ``...``)
initial_re : regular expression, optional
A regular expression matching only the initial prompt, but not continuation.
If no initial expression is given, prompt_re will be used everywhere.
Used mainly for plain Python prompts (``>>>``), where the continuation prompt
``...`` is a valid Python expression in Python 3, so shouldn't be stripped.
Notes
-----
If initial_re and prompt_re differ,
only initial_re will be tested against the first line.
If any prompt is found on the first two lines,
prompts will be stripped from the rest of the block.
"""
def __init__(self, prompt_re, initial_re=None):
self.prompt_re = prompt_re
self.initial_re = initial_re or prompt_re
def _strip(self, lines):
return [self.prompt_re.sub('', l, count=1) for l in lines]
def __call__(self, lines):
if not lines:
return lines
if self.initial_re.match(lines[0]) or \
(len(lines) > 1 and self.prompt_re.match(lines[1])):
return self._strip(lines)
return lines
classic_prompt = PromptStripper(
prompt_re=re.compile(r'^(>>>|\.\.\.)( |$)'),
initial_re=re.compile(r'^>>>( |$)')
)
ipython_prompt = PromptStripper(
re.compile(
r"""
^( # Match from the beginning of a line, either:
# 1. First-line prompt:
((\[nav\]|\[ins\])?\ )? # Vi editing mode prompt, if it's there
In\ # The 'In' of the prompt, with a space
\[\d+\]: # Command index, as displayed in the prompt
\ # With a mandatory trailing space
| # ... or ...
# 2. The three dots of the multiline prompt
\s* # All leading whitespace characters
\.{3,}: # The three (or more) dots
\ ? # With an optional trailing space
)
""",
re.VERBOSE,
)
)
def cell_magic(lines):
if not lines or not lines[0].startswith('%%'):
return lines
if re.match(r'%%\w+\?', lines[0]):
# This case will be handled by help_end
return lines
magic_name, _, first_line = lines[0][2:].rstrip().partition(' ')
body = ''.join(lines[1:])
return ['get_ipython().run_cell_magic(%r, %r, %r)\n'
% (magic_name, first_line, body)]
def _find_assign_op(token_line) -> Optional[int]:
"""Get the index of the first assignment in the line ('=' not inside brackets)
Note: We don't try to support multiple special assignment (a = b = %foo)
"""
paren_level = 0
for i, ti in enumerate(token_line):
s = ti.string
if s == '=' and paren_level == 0:
return i
if s in {'(','[','{'}:
paren_level += 1
elif s in {')', ']', '}'}:
if paren_level > 0:
paren_level -= 1
return None
def find_end_of_continued_line(lines, start_line: int):
"""Find the last line of a line explicitly extended using backslashes.
Uses 0-indexed line numbers.
"""
end_line = start_line
while lines[end_line].endswith('\\\n'):
end_line += 1
if end_line >= len(lines):
break
return end_line
def assemble_continued_line(lines, start: Tuple[int, int], end_line: int):
r"""Assemble a single line from multiple continued line pieces
Continued lines are lines ending in ``\``, and the line following the last
``\`` in the block.
For example, this code continues over multiple lines::
if (assign_ix is not None) \
and (len(line) >= assign_ix + 2) \
and (line[assign_ix+1].string == '%') \
and (line[assign_ix+2].type == tokenize.NAME):
This statement contains four continued line pieces.
Assembling these pieces into a single line would give::
if (assign_ix is not None) and (len(line) >= assign_ix + 2) and (line[...
This uses 0-indexed line numbers. *start* is (lineno, colno).
Used to allow ``%magic`` and ``!system`` commands to be continued over
multiple lines.
"""
parts = [lines[start[0]][start[1]:]] + lines[start[0]+1:end_line+1]
return ' '.join([p.rstrip()[:-1] for p in parts[:-1]] # Strip backslash+newline
+ [parts[-1].rstrip()]) # Strip newline from last line
class TokenTransformBase:
"""Base class for transformations which examine tokens.
Special syntax should not be transformed when it occurs inside strings or
comments. This is hard to reliably avoid with regexes. The solution is to
tokenise the code as Python, and recognise the special syntax in the tokens.
IPython's special syntax is not valid Python syntax, so tokenising may go
wrong after the special syntax starts. These classes therefore find and
transform *one* instance of special syntax at a time into regular Python
syntax. After each transformation, tokens are regenerated to find the next
piece of special syntax.
Subclasses need to implement one class method (find)
and one regular method (transform).
The priority attribute can select which transformation to apply if multiple
transformers match in the same place. Lower numbers have higher priority.
This allows "%magic?" to be turned into a help call rather than a magic call.
"""
# Lower numbers -> higher priority (for matches in the same location)
priority = 10
def sortby(self):
return self.start_line, self.start_col, self.priority
def __init__(self, start):
self.start_line = start[0] - 1 # Shift from 1-index to 0-index
self.start_col = start[1]
@classmethod
def find(cls, tokens_by_line):
"""Find one instance of special syntax in the provided tokens.
Tokens are grouped into logical lines for convenience,
so it is easy to e.g. look at the first token of each line.
*tokens_by_line* is a list of lists of tokenize.TokenInfo objects.
This should return an instance of its class, pointing to the start
position it has found, or None if it found no match.
"""
raise NotImplementedError
def transform(self, lines: List[str]):
"""Transform one instance of special syntax found by ``find()``
Takes a list of strings representing physical lines,
returns a similar list of transformed lines.
"""
raise NotImplementedError
class MagicAssign(TokenTransformBase):
"""Transformer for assignments from magics (a = %foo)"""
@classmethod
def find(cls, tokens_by_line):
"""Find the first magic assignment (a = %foo) in the cell.
"""
for line in tokens_by_line:
assign_ix = _find_assign_op(line)
if (assign_ix is not None) \
and (len(line) >= assign_ix + 2) \
and (line[assign_ix+1].string == '%') \
and (line[assign_ix+2].type == tokenize.NAME):
return cls(line[assign_ix+1].start)
def transform(self, lines: List[str]):
"""Transform a magic assignment found by the ``find()`` classmethod.
"""
start_line, start_col = self.start_line, self.start_col
lhs = lines[start_line][:start_col]
end_line = find_end_of_continued_line(lines, start_line)
rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
assert rhs.startswith('%'), rhs
magic_name, _, args = rhs[1:].partition(' ')
lines_before = lines[:start_line]
call = "get_ipython().run_line_magic({!r}, {!r})".format(magic_name, args)
new_line = lhs + call + '\n'
lines_after = lines[end_line+1:]
return lines_before + [new_line] + lines_after
class SystemAssign(TokenTransformBase):
"""Transformer for assignments from system commands (a = !foo)"""
@classmethod
def find(cls, tokens_by_line):
"""Find the first system assignment (a = !foo) in the cell.
"""
for line in tokens_by_line:
assign_ix = _find_assign_op(line)
if (assign_ix is not None) \
and not line[assign_ix].line.strip().startswith('=') \
and (len(line) >= assign_ix + 2) \
and (line[assign_ix + 1].type == tokenize.ERRORTOKEN):
ix = assign_ix + 1
while ix < len(line) and line[ix].type == tokenize.ERRORTOKEN:
if line[ix].string == '!':
return cls(line[ix].start)
elif not line[ix].string.isspace():
break
ix += 1
def transform(self, lines: List[str]):
"""Transform a system assignment found by the ``find()`` classmethod.
"""
start_line, start_col = self.start_line, self.start_col
lhs = lines[start_line][:start_col]
end_line = find_end_of_continued_line(lines, start_line)
rhs = assemble_continued_line(lines, (start_line, start_col), end_line)
assert rhs.startswith('!'), rhs
cmd = rhs[1:]
lines_before = lines[:start_line]
call = "get_ipython().getoutput({!r})".format(cmd)
new_line = lhs + call + '\n'
lines_after = lines[end_line + 1:]
return lines_before + [new_line] + lines_after
# The escape sequences that define the syntax transformations IPython will
# apply to user input. These can NOT be just changed here: many regular
# expressions and other parts of the code may use their hardcoded values, and
# for all intents and purposes they constitute the 'IPython syntax', so they
# should be considered fixed.
ESC_SHELL = '!' # Send line to underlying system shell
ESC_SH_CAP = '!!' # Send line to system shell and capture output
ESC_HELP = '?' # Find information about object
ESC_HELP2 = '??' # Find extra-detailed information about object
ESC_MAGIC = '%' # Call magic function
ESC_MAGIC2 = '%%' # Call cell-magic function
ESC_QUOTE = ',' # Split args on whitespace, quote each as string and call
ESC_QUOTE2 = ';' # Quote all args as a single string, call
ESC_PAREN = '/' # Call first argument with rest of line as arguments
ESCAPE_SINGLES = {'!', '?', '%', ',', ';', '/'}
ESCAPE_DOUBLES = {'!!', '??'} # %% (cell magic) is handled separately
def _make_help_call(target, esc):
"""Prepares a pinfo(2)/psearch call from a target name and the escape
(i.e. ? or ??)"""
method = 'pinfo2' if esc == '??' \
else 'psearch' if '*' in target \
else 'pinfo'
arg = " ".join([method, target])
#Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
t_magic_name, _, t_magic_arg_s = arg.partition(' ')
t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
return "get_ipython().run_line_magic(%r, %r)" % (t_magic_name, t_magic_arg_s)
def _tr_help(content):
"""Translate lines escaped with: ?
A naked help line should fire the intro help screen (shell.show_usage())
"""
if not content:
return 'get_ipython().show_usage()'
return _make_help_call(content, '?')
def _tr_help2(content):
"""Translate lines escaped with: ??
A naked help line should fire the intro help screen (shell.show_usage())
"""
if not content:
return 'get_ipython().show_usage()'
return _make_help_call(content, '??')
def _tr_magic(content):
"Translate lines escaped with a percent sign: %"
name, _, args = content.partition(' ')
return 'get_ipython().run_line_magic(%r, %r)' % (name, args)
def _tr_quote(content):
"Translate lines escaped with a comma: ,"
name, _, args = content.partition(' ')
return '%s("%s")' % (name, '", "'.join(args.split()) )
def _tr_quote2(content):
"Translate lines escaped with a semicolon: ;"
name, _, args = content.partition(' ')
return '%s("%s")' % (name, args)
def _tr_paren(content):
"Translate lines escaped with a slash: /"
name, _, args = content.partition(' ')
return '%s(%s)' % (name, ", ".join(args.split()))
tr = { ESC_SHELL : 'get_ipython().system({!r})'.format,
ESC_SH_CAP : 'get_ipython().getoutput({!r})'.format,
ESC_HELP : _tr_help,
ESC_HELP2 : _tr_help2,
ESC_MAGIC : _tr_magic,
ESC_QUOTE : _tr_quote,
ESC_QUOTE2 : _tr_quote2,
ESC_PAREN : _tr_paren }
class EscapedCommand(TokenTransformBase):
"""Transformer for escaped commands like %foo, !foo, or /foo"""
@classmethod
def find(cls, tokens_by_line):
"""Find the first escaped command (%foo, !foo, etc.) in the cell.
"""
for line in tokens_by_line:
if not line:
continue
ix = 0
ll = len(line)
while ll > ix and line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
ix += 1
if ix >= ll:
continue
if line[ix].string in ESCAPE_SINGLES:
return cls(line[ix].start)
def transform(self, lines):
"""Transform an escaped line found by the ``find()`` classmethod.
"""
start_line, start_col = self.start_line, self.start_col
indent = lines[start_line][:start_col]
end_line = find_end_of_continued_line(lines, start_line)
line = assemble_continued_line(lines, (start_line, start_col), end_line)
if len(line) > 1 and line[:2] in ESCAPE_DOUBLES:
escape, content = line[:2], line[2:]
else:
escape, content = line[:1], line[1:]
if escape in tr:
call = tr[escape](content)
else:
call = ''
lines_before = lines[:start_line]
new_line = indent + call + '\n'
lines_after = lines[end_line + 1:]
return lines_before + [new_line] + lines_after
_help_end_re = re.compile(r"""(%{0,2}
(?!\d)[\w*]+ # Variable name
(\.(?!\d)[\w*]+)* # .etc.etc
)
(\?\??)$ # ? or ??
""",
re.VERBOSE)
class HelpEnd(TokenTransformBase):
"""Transformer for help syntax: obj? and obj??"""
# This needs to be higher priority (lower number) than EscapedCommand so
# that inspecting magics (%foo?) works.
priority = 5
def __init__(self, start, q_locn):
super().__init__(start)
self.q_line = q_locn[0] - 1 # Shift from 1-indexed to 0-indexed
self.q_col = q_locn[1]
@classmethod
def find(cls, tokens_by_line):
"""Find the first help command (foo?) in the cell.
"""
for line in tokens_by_line:
# Last token is NEWLINE; look at last but one
if len(line) > 2 and line[-2].string == '?':
# Find the first token that's not INDENT/DEDENT
ix = 0
while line[ix].type in {tokenize.INDENT, tokenize.DEDENT}:
ix += 1
return cls(line[ix].start, line[-2].start)
def transform(self, lines):
"""Transform a help command found by the ``find()`` classmethod.
"""
piece = ''.join(lines[self.start_line:self.q_line+1])
indent, content = piece[:self.start_col], piece[self.start_col:]
lines_before = lines[:self.start_line]
lines_after = lines[self.q_line + 1:]
m = _help_end_re.search(content)
if not m:
raise SyntaxError(content)
assert m is not None, content
target = m.group(1)
esc = m.group(3)
call = _make_help_call(target, esc)
new_line = indent + call + '\n'
return lines_before + [new_line] + lines_after
def make_tokens_by_line(lines:List[str]):
"""Tokenize a series of lines and group tokens by line.
The tokens for a multiline Python string or expression are grouped as one
line. All lines except the last lines should keep their line ending ('\\n',
'\\r\\n') for this to properly work. Use `.splitlines(keeplineending=True)`
for example when passing block of text to this function.
"""
# NL tokens are used inside multiline expressions, but also after blank
# lines or comments. This is intentional - see https://bugs.python.org/issue17061
# We want to group the former case together but split the latter, so we
# track parentheses level, similar to the internals of tokenize.
# reexported from token on 3.7+
NEWLINE, NL = tokenize.NEWLINE, tokenize.NL # type: ignore
tokens_by_line: List[List[Any]] = [[]]
if len(lines) > 1 and not lines[0].endswith(("\n", "\r", "\r\n", "\x0b", "\x0c")):
warnings.warn(
"`make_tokens_by_line` received a list of lines which do not have lineending markers ('\\n', '\\r', '\\r\\n', '\\x0b', '\\x0c'), behavior will be unspecified",
stacklevel=2,
)
parenlev = 0
try:
for token in tokenize.generate_tokens(iter(lines).__next__):
tokens_by_line[-1].append(token)
if (token.type == NEWLINE) \
or ((token.type == NL) and (parenlev <= 0)):
tokens_by_line.append([])
elif token.string in {'(', '[', '{'}:
parenlev += 1
elif token.string in {')', ']', '}'}:
if parenlev > 0:
parenlev -= 1
except tokenize.TokenError:
# Input ended in a multiline string or expression. That's OK for us.
pass
if not tokens_by_line[-1]:
tokens_by_line.pop()
return tokens_by_line
def has_sunken_brackets(tokens: List[tokenize.TokenInfo]):
"""Check if the depth of brackets in the list of tokens drops below 0"""
parenlev = 0
for token in tokens:
if token.string in {"(", "[", "{"}:
parenlev += 1
elif token.string in {")", "]", "}"}:
parenlev -= 1
if parenlev < 0:
return True
return False
def show_linewise_tokens(s: str):
"""For investigation and debugging"""
if not s.endswith('\n'):
s += '\n'
lines = s.splitlines(keepends=True)
for line in make_tokens_by_line(lines):
print("Line -------")
for tokinfo in line:
print(" ", tokinfo)
# Arbitrary limit to prevent getting stuck in infinite loops
TRANSFORM_LOOP_LIMIT = 500
class TransformerManager:
"""Applies various transformations to a cell or code block.
The key methods for external use are ``transform_cell()``
and ``check_complete()``.
"""
def __init__(self):
self.cleanup_transforms = [
leading_empty_lines,
leading_indent,
classic_prompt,
ipython_prompt,
]
self.line_transforms = [
cell_magic,
]
self.token_transformers = [
MagicAssign,
SystemAssign,
EscapedCommand,
HelpEnd,
]
def do_one_token_transform(self, lines):
"""Find and run the transform earliest in the code.
Returns (changed, lines).
This method is called repeatedly until changed is False, indicating
that all available transformations are complete.
The tokens following IPython special syntax might not be valid, so
the transformed code is retokenised every time to identify the next
piece of special syntax. Hopefully long code cells are mostly valid
Python, not using lots of IPython special syntax, so this shouldn't be
a performance issue.
"""
tokens_by_line = make_tokens_by_line(lines)
candidates = []
for transformer_cls in self.token_transformers:
transformer = transformer_cls.find(tokens_by_line)
if transformer:
candidates.append(transformer)
if not candidates:
# Nothing to transform
return False, lines
ordered_transformers = sorted(candidates, key=TokenTransformBase.sortby)
for transformer in ordered_transformers:
try:
return True, transformer.transform(lines)
except SyntaxError:
pass
return False, lines
def do_token_transforms(self, lines):
for _ in range(TRANSFORM_LOOP_LIMIT):
changed, lines = self.do_one_token_transform(lines)
if not changed:
return lines
raise RuntimeError("Input transformation still changing after "
"%d iterations. Aborting." % TRANSFORM_LOOP_LIMIT)
def transform_cell(self, cell: str) -> str:
"""Transforms a cell of input code"""
if not cell.endswith('\n'):
cell += '\n' # Ensure the cell has a trailing newline
lines = cell.splitlines(keepends=True)
for transform in self.cleanup_transforms + self.line_transforms:
lines = transform(lines)
lines = self.do_token_transforms(lines)
return ''.join(lines)
def check_complete(self, cell: str):
"""Return whether a block of code is ready to execute, or should be continued
Parameters
----------
cell : string
Python input code, which can be multiline.
Returns
-------
status : str
One of 'complete', 'incomplete', or 'invalid' if source is not a
prefix of valid code.
indent_spaces : int or None
The number of spaces by which to indent the next line of code. If
status is not 'incomplete', this is None.
"""
# Remember if the lines ends in a new line.
ends_with_newline = False
for character in reversed(cell):
if character == '\n':
ends_with_newline = True
break
elif character.strip():
break
else:
continue
if not ends_with_newline:
# Append an newline for consistent tokenization
# See https://bugs.python.org/issue33899
cell += '\n'
lines = cell.splitlines(keepends=True)
if not lines:
return 'complete', None
if lines[-1].endswith('\\'):
# Explicit backslash continuation
return 'incomplete', find_last_indent(lines)
try:
for transform in self.cleanup_transforms:
if not getattr(transform, 'has_side_effects', False):
lines = transform(lines)
except SyntaxError:
return 'invalid', None
if lines[0].startswith('%%'):
# Special case for cell magics - completion marked by blank line
if lines[-1].strip():
return 'incomplete', find_last_indent(lines)
else:
return 'complete', None
try:
for transform in self.line_transforms:
if not getattr(transform, 'has_side_effects', False):
lines = transform(lines)
lines = self.do_token_transforms(lines)
except SyntaxError:
return 'invalid', None
tokens_by_line = make_tokens_by_line(lines)
# Bail if we got one line and there are more closing parentheses than
# the opening ones
if (
len(lines) == 1
and tokens_by_line
and has_sunken_brackets(tokens_by_line[0])
):
return "invalid", None
if not tokens_by_line:
return 'incomplete', find_last_indent(lines)
if tokens_by_line[-1][-1].type != tokenize.ENDMARKER:
# We're in a multiline string or expression
return 'incomplete', find_last_indent(lines)
newline_types = {tokenize.NEWLINE, tokenize.COMMENT, tokenize.ENDMARKER} # type: ignore
# Pop the last line which only contains DEDENTs and ENDMARKER
last_token_line = None
if {t.type for t in tokens_by_line[-1]} in [
{tokenize.DEDENT, tokenize.ENDMARKER},
{tokenize.ENDMARKER}
] and len(tokens_by_line) > 1:
last_token_line = tokens_by_line.pop()
while tokens_by_line[-1] and tokens_by_line[-1][-1].type in newline_types:
tokens_by_line[-1].pop()
if not tokens_by_line[-1]:
return 'incomplete', find_last_indent(lines)
if tokens_by_line[-1][-1].string == ':':
# The last line starts a block (e.g. 'if foo:')
ix = 0
while tokens_by_line[-1][ix].type in {tokenize.INDENT, tokenize.DEDENT}:
ix += 1
indent = tokens_by_line[-1][ix].start[1]
return 'incomplete', indent + 4
if tokens_by_line[-1][0].line.endswith('\\'):
return 'incomplete', None
# At this point, our checks think the code is complete (or invalid).
# We'll use codeop.compile_command to check this with the real parser
try:
with warnings.catch_warnings():
warnings.simplefilter('error', SyntaxWarning)
res = compile_command(''.join(lines), symbol='exec')
except (SyntaxError, OverflowError, ValueError, TypeError,
MemoryError, SyntaxWarning):
return 'invalid', None
else:
if res is None:
return 'incomplete', find_last_indent(lines)
if last_token_line and last_token_line[0].type == tokenize.DEDENT:
if ends_with_newline:
return 'complete', None
return 'incomplete', find_last_indent(lines)
# If there's a blank line at the end, assume we're ready to execute
if not lines[-1].strip():
return 'complete', None
return 'complete', None
def find_last_indent(lines):
m = _indent_re.match(lines[-1])
if not m:
return 0
return len(m.group(0).replace('\t', ' '*4))
class MaybeAsyncCompile(Compile):
def __init__(self, extra_flags=0):
super().__init__()
self.flags |= extra_flags
class MaybeAsyncCommandCompiler(CommandCompiler):
def __init__(self, extra_flags=0):
self.compiler = MaybeAsyncCompile(extra_flags=extra_flags)
_extra_flags = ast.PyCF_ALLOW_TOP_LEVEL_AWAIT
compile_command = MaybeAsyncCommandCompiler(extra_flags=_extra_flags)

@ -0,0 +1,218 @@
"""Logger class for IPython's logging facilities.
"""
#*****************************************************************************
# Copyright (C) 2001 Janko Hauser <jhauser@zscout.de> and
# Copyright (C) 2001-2006 Fernando Perez <fperez@colorado.edu>
#
# Distributed under the terms of the BSD License. The full license is in
# the file COPYING, distributed as part of this software.
#*****************************************************************************
#****************************************************************************
# Modules and globals
# Python standard modules
import glob
import io
import os
import time
#****************************************************************************
# FIXME: This class isn't a mixin anymore, but it still needs attributes from
# ipython and does input cache management. Finish cleanup later...
class Logger(object):
"""A Logfile class with different policies for file creation"""
def __init__(self, home_dir, logfname='Logger.log', loghead=u'',
logmode='over'):
# this is the full ipython instance, we need some attributes from it
# which won't exist until later. What a mess, clean up later...
self.home_dir = home_dir
self.logfname = logfname
self.loghead = loghead
self.logmode = logmode
self.logfile = None
# Whether to log raw or processed input
self.log_raw_input = False
# whether to also log output
self.log_output = False
# whether to put timestamps before each log entry
self.timestamp = False
# activity control flags
self.log_active = False
# logmode is a validated property
def _set_mode(self,mode):
if mode not in ['append','backup','global','over','rotate']:
raise ValueError('invalid log mode %s given' % mode)
self._logmode = mode
def _get_mode(self):
return self._logmode
logmode = property(_get_mode,_set_mode)
def logstart(self, logfname=None, loghead=None, logmode=None,
log_output=False, timestamp=False, log_raw_input=False):
"""Generate a new log-file with a default header.
Raises RuntimeError if the log has already been started"""
if self.logfile is not None:
raise RuntimeError('Log file is already active: %s' %
self.logfname)
# The parameters can override constructor defaults
if logfname is not None: self.logfname = logfname
if loghead is not None: self.loghead = loghead
if logmode is not None: self.logmode = logmode
# Parameters not part of the constructor
self.timestamp = timestamp
self.log_output = log_output
self.log_raw_input = log_raw_input
# init depending on the log mode requested
isfile = os.path.isfile
logmode = self.logmode
if logmode == 'append':
self.logfile = io.open(self.logfname, 'a', encoding='utf-8')
elif logmode == 'backup':
if isfile(self.logfname):
backup_logname = self.logfname+'~'
# Manually remove any old backup, since os.rename may fail
# under Windows.
if isfile(backup_logname):
os.remove(backup_logname)
os.rename(self.logfname,backup_logname)
self.logfile = io.open(self.logfname, 'w', encoding='utf-8')
elif logmode == 'global':
self.logfname = os.path.join(self.home_dir,self.logfname)
self.logfile = io.open(self.logfname, 'a', encoding='utf-8')
elif logmode == 'over':
if isfile(self.logfname):
os.remove(self.logfname)
self.logfile = io.open(self.logfname,'w', encoding='utf-8')
elif logmode == 'rotate':
if isfile(self.logfname):
if isfile(self.logfname+'.001~'):
old = glob.glob(self.logfname+'.*~')
old.sort()
old.reverse()
for f in old:
root, ext = os.path.splitext(f)
num = int(ext[1:-1])+1
os.rename(f, root+'.'+repr(num).zfill(3)+'~')
os.rename(self.logfname, self.logfname+'.001~')
self.logfile = io.open(self.logfname, 'w', encoding='utf-8')
if logmode != 'append':
self.logfile.write(self.loghead)
self.logfile.flush()
self.log_active = True
def switch_log(self,val):
"""Switch logging on/off. val should be ONLY a boolean."""
if val not in [False,True,0,1]:
raise ValueError('Call switch_log ONLY with a boolean argument, '
'not with: %s' % val)
label = {0:'OFF',1:'ON',False:'OFF',True:'ON'}
if self.logfile is None:
print("""
Logging hasn't been started yet (use logstart for that).
%logon/%logoff are for temporarily starting and stopping logging for a logfile
which already exists. But you must first start the logging process with
%logstart (optionally giving a logfile name).""")
else:
if self.log_active == val:
print('Logging is already',label[val])
else:
print('Switching logging',label[val])
self.log_active = not self.log_active
self.log_active_out = self.log_active
def logstate(self):
"""Print a status message about the logger."""
if self.logfile is None:
print('Logging has not been activated.')
else:
state = self.log_active and 'active' or 'temporarily suspended'
print('Filename :', self.logfname)
print('Mode :', self.logmode)
print('Output logging :', self.log_output)
print('Raw input log :', self.log_raw_input)
print('Timestamping :', self.timestamp)
print('State :', state)
def log(self, line_mod, line_ori):
"""Write the sources to a log.
Inputs:
- line_mod: possibly modified input, such as the transformations made
by input prefilters or input handlers of various kinds. This should
always be valid Python.
- line_ori: unmodified input line from the user. This is not
necessarily valid Python.
"""
# Write the log line, but decide which one according to the
# log_raw_input flag, set when the log is started.
if self.log_raw_input:
self.log_write(line_ori)
else:
self.log_write(line_mod)
def log_write(self, data, kind='input'):
"""Write data to the log file, if active"""
#print 'data: %r' % data # dbg
if self.log_active and data:
write = self.logfile.write
if kind=='input':
if self.timestamp:
write(time.strftime('# %a, %d %b %Y %H:%M:%S\n', time.localtime()))
write(data)
elif kind=='output' and self.log_output:
odata = u'\n'.join([u'#[Out]# %s' % s
for s in data.splitlines()])
write(u'%s\n' % odata)
self.logfile.flush()
def logstop(self):
"""Fully stop logging and close log file.
In order to start logging again, a new logstart() call needs to be
made, possibly (though not necessarily) with a new filename, mode and
other options."""
if self.logfile is not None:
self.logfile.close()
self.logfile = None
else:
print("Logging hadn't been started.")
self.log_active = False
# For backwards compatibility, in case anyone was using this.
close_log = logstop

@ -0,0 +1,53 @@
"""Support for interactive macros in IPython"""
#*****************************************************************************
# Copyright (C) 2001-2005 Fernando Perez <fperez@colorado.edu>
#
# Distributed under the terms of the BSD License. The full license is in
# the file COPYING, distributed as part of this software.
#*****************************************************************************
import re
from IPython.utils.encoding import DEFAULT_ENCODING
coding_declaration = re.compile(r"#\s*coding[:=]\s*([-\w.]+)")
class Macro(object):
"""Simple class to store the value of macros as strings.
Macro is just a callable that executes a string of IPython
input when called.
"""
def __init__(self,code):
"""store the macro value, as a single string which can be executed"""
lines = []
enc = None
for line in code.splitlines():
coding_match = coding_declaration.match(line)
if coding_match:
enc = coding_match.group(1)
else:
lines.append(line)
code = "\n".join(lines)
if isinstance(code, bytes):
code = code.decode(enc or DEFAULT_ENCODING)
self.value = code + '\n'
def __str__(self):
return self.value
def __repr__(self):
return 'IPython.macro.Macro(%s)' % repr(self.value)
def __getstate__(self):
""" needed for safe pickling via %store """
return {'value': self.value}
def __add__(self, other):
if isinstance(other, Macro):
return Macro(self.value + other.value)
elif isinstance(other, str):
return Macro(self.value + other)
raise TypeError

@ -0,0 +1,746 @@
# encoding: utf-8
"""Magic functions for InteractiveShell.
"""
#-----------------------------------------------------------------------------
# Copyright (C) 2001 Janko Hauser <jhauser@zscout.de> and
# Copyright (C) 2001 Fernando Perez <fperez@colorado.edu>
# Copyright (C) 2008 The IPython Development Team
# Distributed under the terms of the BSD License. The full license is in
# the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------
import os
import re
import sys
from getopt import getopt, GetoptError
from traitlets.config.configurable import Configurable
from . import oinspect
from .error import UsageError
from .inputtransformer2 import ESC_MAGIC, ESC_MAGIC2
from ..utils.ipstruct import Struct
from ..utils.process import arg_split
from ..utils.text import dedent
from traitlets import Bool, Dict, Instance, observe
from logging import error
#-----------------------------------------------------------------------------
# Globals
#-----------------------------------------------------------------------------
# A dict we'll use for each class that has magics, used as temporary storage to
# pass information between the @line/cell_magic method decorators and the
# @magics_class class decorator, because the method decorators have no
# access to the class when they run. See for more details:
# http://stackoverflow.com/questions/2366713/can-a-python-decorator-of-an-instance-method-access-the-class
magics = dict(line={}, cell={})
magic_kinds = ('line', 'cell')
magic_spec = ('line', 'cell', 'line_cell')
magic_escapes = dict(line=ESC_MAGIC, cell=ESC_MAGIC2)
#-----------------------------------------------------------------------------
# Utility classes and functions
#-----------------------------------------------------------------------------
class Bunch: pass
def on_off(tag):
"""Return an ON/OFF string for a 1/0 input. Simple utility function."""
return ['OFF','ON'][tag]
def compress_dhist(dh):
"""Compress a directory history into a new one with at most 20 entries.
Return a new list made from the first and last 10 elements of dhist after
removal of duplicates.
"""
head, tail = dh[:-10], dh[-10:]
newhead = []
done = set()
for h in head:
if h in done:
continue
newhead.append(h)
done.add(h)
return newhead + tail
def needs_local_scope(func):
"""Decorator to mark magic functions which need to local scope to run."""
func.needs_local_scope = True
return func
#-----------------------------------------------------------------------------
# Class and method decorators for registering magics
#-----------------------------------------------------------------------------
def magics_class(cls):
"""Class decorator for all subclasses of the main Magics class.
Any class that subclasses Magics *must* also apply this decorator, to
ensure that all the methods that have been decorated as line/cell magics
get correctly registered in the class instance. This is necessary because
when method decorators run, the class does not exist yet, so they
temporarily store their information into a module global. Application of
this class decorator copies that global data to the class instance and
clears the global.
Obviously, this mechanism is not thread-safe, which means that the
*creation* of subclasses of Magic should only be done in a single-thread
context. Instantiation of the classes has no restrictions. Given that
these classes are typically created at IPython startup time and before user
application code becomes active, in practice this should not pose any
problems.
"""
cls.registered = True
cls.magics = dict(line = magics['line'],
cell = magics['cell'])
magics['line'] = {}
magics['cell'] = {}
return cls
def record_magic(dct, magic_kind, magic_name, func):
"""Utility function to store a function as a magic of a specific kind.
Parameters
----------
dct : dict
A dictionary with 'line' and 'cell' subdicts.
magic_kind : str
Kind of magic to be stored.
magic_name : str
Key to store the magic as.
func : function
Callable object to store.
"""
if magic_kind == 'line_cell':
dct['line'][magic_name] = dct['cell'][magic_name] = func
else:
dct[magic_kind][magic_name] = func
def validate_type(magic_kind):
"""Ensure that the given magic_kind is valid.
Check that the given magic_kind is one of the accepted spec types (stored
in the global `magic_spec`), raise ValueError otherwise.
"""
if magic_kind not in magic_spec:
raise ValueError('magic_kind must be one of %s, %s given' %
magic_kinds, magic_kind)
# The docstrings for the decorator below will be fairly similar for the two
# types (method and function), so we generate them here once and reuse the
# templates below.
_docstring_template = \
"""Decorate the given {0} as {1} magic.
The decorator can be used with or without arguments, as follows.
i) without arguments: it will create a {1} magic named as the {0} being
decorated::
@deco
def foo(...)
will create a {1} magic named `foo`.
ii) with one string argument: which will be used as the actual name of the
resulting magic::
@deco('bar')
def foo(...)
will create a {1} magic named `bar`.
To register a class magic use ``Interactiveshell.register_magic(class or instance)``.
"""
# These two are decorator factories. While they are conceptually very similar,
# there are enough differences in the details that it's simpler to have them
# written as completely standalone functions rather than trying to share code
# and make a single one with convoluted logic.
def _method_magic_marker(magic_kind):
"""Decorator factory for methods in Magics subclasses.
"""
validate_type(magic_kind)
# This is a closure to capture the magic_kind. We could also use a class,
# but it's overkill for just that one bit of state.
def magic_deco(arg):
if callable(arg):
# "Naked" decorator call (just @foo, no args)
func = arg
name = func.__name__
retval = arg
record_magic(magics, magic_kind, name, name)
elif isinstance(arg, str):
# Decorator called with arguments (@foo('bar'))
name = arg
def mark(func, *a, **kw):
record_magic(magics, magic_kind, name, func.__name__)
return func
retval = mark
else:
raise TypeError("Decorator can only be called with "
"string or function")
return retval
# Ensure the resulting decorator has a usable docstring
magic_deco.__doc__ = _docstring_template.format('method', magic_kind)
return magic_deco
def _function_magic_marker(magic_kind):
"""Decorator factory for standalone functions.
"""
validate_type(magic_kind)
# This is a closure to capture the magic_kind. We could also use a class,
# but it's overkill for just that one bit of state.
def magic_deco(arg):
# Find get_ipython() in the caller's namespace
caller = sys._getframe(1)
for ns in ['f_locals', 'f_globals', 'f_builtins']:
get_ipython = getattr(caller, ns).get('get_ipython')
if get_ipython is not None:
break
else:
raise NameError('Decorator can only run in context where '
'`get_ipython` exists')
ip = get_ipython()
if callable(arg):
# "Naked" decorator call (just @foo, no args)
func = arg
name = func.__name__
ip.register_magic_function(func, magic_kind, name)
retval = arg
elif isinstance(arg, str):
# Decorator called with arguments (@foo('bar'))
name = arg
def mark(func, *a, **kw):
ip.register_magic_function(func, magic_kind, name)
return func
retval = mark
else:
raise TypeError("Decorator can only be called with "
"string or function")
return retval
# Ensure the resulting decorator has a usable docstring
ds = _docstring_template.format('function', magic_kind)
ds += dedent("""
Note: this decorator can only be used in a context where IPython is already
active, so that the `get_ipython()` call succeeds. You can therefore use
it in your startup files loaded after IPython initializes, but *not* in the
IPython configuration file itself, which is executed before IPython is
fully up and running. Any file located in the `startup` subdirectory of
your configuration profile will be OK in this sense.
""")
magic_deco.__doc__ = ds
return magic_deco
MAGIC_NO_VAR_EXPAND_ATTR = '_ipython_magic_no_var_expand'
def no_var_expand(magic_func):
"""Mark a magic function as not needing variable expansion
By default, IPython interprets `{a}` or `$a` in the line passed to magics
as variables that should be interpolated from the interactive namespace
before passing the line to the magic function.
This is not always desirable, e.g. when the magic executes Python code
(%timeit, %time, etc.).
Decorate magics with `@no_var_expand` to opt-out of variable expansion.
.. versionadded:: 7.3
"""
setattr(magic_func, MAGIC_NO_VAR_EXPAND_ATTR, True)
return magic_func
# Create the actual decorators for public use
# These three are used to decorate methods in class definitions
line_magic = _method_magic_marker('line')
cell_magic = _method_magic_marker('cell')
line_cell_magic = _method_magic_marker('line_cell')
# These three decorate standalone functions and perform the decoration
# immediately. They can only run where get_ipython() works
register_line_magic = _function_magic_marker('line')
register_cell_magic = _function_magic_marker('cell')
register_line_cell_magic = _function_magic_marker('line_cell')
#-----------------------------------------------------------------------------
# Core Magic classes
#-----------------------------------------------------------------------------
class MagicsManager(Configurable):
"""Object that handles all magic-related functionality for IPython.
"""
# Non-configurable class attributes
# A two-level dict, first keyed by magic type, then by magic function, and
# holding the actual callable object as value. This is the dict used for
# magic function dispatch
magics = Dict()
lazy_magics = Dict(
help="""
Mapping from magic names to modules to load.
This can be used in IPython/IPykernel configuration to declare lazy magics
that will only be imported/registered on first use.
For example::
c.MagicsManager.lazy_magics = {
"my_magic": "slow.to.import",
"my_other_magic": "also.slow",
}
On first invocation of `%my_magic`, `%%my_magic`, `%%my_other_magic` or
`%%my_other_magic`, the corresponding module will be loaded as an ipython
extensions as if you had previously done `%load_ext ipython`.
Magics names should be without percent(s) as magics can be both cell
and line magics.
Lazy loading happen relatively late in execution process, and
complex extensions that manipulate Python/IPython internal state or global state
might not support lazy loading.
"""
).tag(
config=True,
)
# A registry of the original objects that we've been given holding magics.
registry = Dict()
shell = Instance('IPython.core.interactiveshell.InteractiveShellABC', allow_none=True)
auto_magic = Bool(True, help=
"Automatically call line magics without requiring explicit % prefix"
).tag(config=True)
@observe('auto_magic')
def _auto_magic_changed(self, change):
self.shell.automagic = change['new']
_auto_status = [
'Automagic is OFF, % prefix IS needed for line magics.',
'Automagic is ON, % prefix IS NOT needed for line magics.']
user_magics = Instance('IPython.core.magics.UserMagics', allow_none=True)
def __init__(self, shell=None, config=None, user_magics=None, **traits):
super(MagicsManager, self).__init__(shell=shell, config=config,
user_magics=user_magics, **traits)
self.magics = dict(line={}, cell={})
# Let's add the user_magics to the registry for uniformity, so *all*
# registered magic containers can be found there.
self.registry[user_magics.__class__.__name__] = user_magics
def auto_status(self):
"""Return descriptive string with automagic status."""
return self._auto_status[self.auto_magic]
def lsmagic(self):
"""Return a dict of currently available magic functions.
The return dict has the keys 'line' and 'cell', corresponding to the
two types of magics we support. Each value is a list of names.
"""
return self.magics
def lsmagic_docs(self, brief=False, missing=''):
"""Return dict of documentation of magic functions.
The return dict has the keys 'line' and 'cell', corresponding to the
two types of magics we support. Each value is a dict keyed by magic
name whose value is the function docstring. If a docstring is
unavailable, the value of `missing` is used instead.
If brief is True, only the first line of each docstring will be returned.
"""
docs = {}
for m_type in self.magics:
m_docs = {}
for m_name, m_func in self.magics[m_type].items():
if m_func.__doc__:
if brief:
m_docs[m_name] = m_func.__doc__.split('\n', 1)[0]
else:
m_docs[m_name] = m_func.__doc__.rstrip()
else:
m_docs[m_name] = missing
docs[m_type] = m_docs
return docs
def register_lazy(self, name: str, fully_qualified_name: str):
"""
Lazily register a magic via an extension.
Parameters
----------
name : str
Name of the magic you wish to register.
fully_qualified_name :
Fully qualified name of the module/submodule that should be loaded
as an extensions when the magic is first called.
It is assumed that loading this extensions will register the given
magic.
"""
self.lazy_magics[name] = fully_qualified_name
def register(self, *magic_objects):
"""Register one or more instances of Magics.
Take one or more classes or instances of classes that subclass the main
`core.Magic` class, and register them with IPython to use the magic
functions they provide. The registration process will then ensure that
any methods that have decorated to provide line and/or cell magics will
be recognized with the `%x`/`%%x` syntax as a line/cell magic
respectively.
If classes are given, they will be instantiated with the default
constructor. If your classes need a custom constructor, you should
instanitate them first and pass the instance.
The provided arguments can be an arbitrary mix of classes and instances.
Parameters
----------
*magic_objects : one or more classes or instances
"""
# Start by validating them to ensure they have all had their magic
# methods registered at the instance level
for m in magic_objects:
if not m.registered:
raise ValueError("Class of magics %r was constructed without "
"the @register_magics class decorator")
if isinstance(m, type):
# If we're given an uninstantiated class
m = m(shell=self.shell)
# Now that we have an instance, we can register it and update the
# table of callables
self.registry[m.__class__.__name__] = m
for mtype in magic_kinds:
self.magics[mtype].update(m.magics[mtype])
def register_function(self, func, magic_kind='line', magic_name=None):
"""Expose a standalone function as magic function for IPython.
This will create an IPython magic (line, cell or both) from a
standalone function. The functions should have the following
signatures:
* For line magics: `def f(line)`
* For cell magics: `def f(line, cell)`
* For a function that does both: `def f(line, cell=None)`
In the latter case, the function will be called with `cell==None` when
invoked as `%f`, and with cell as a string when invoked as `%%f`.
Parameters
----------
func : callable
Function to be registered as a magic.
magic_kind : str
Kind of magic, one of 'line', 'cell' or 'line_cell'
magic_name : optional str
If given, the name the magic will have in the IPython namespace. By
default, the name of the function itself is used.
"""
# Create the new method in the user_magics and register it in the
# global table
validate_type(magic_kind)
magic_name = func.__name__ if magic_name is None else magic_name
setattr(self.user_magics, magic_name, func)
record_magic(self.magics, magic_kind, magic_name, func)
def register_alias(self, alias_name, magic_name, magic_kind='line', magic_params=None):
"""Register an alias to a magic function.
The alias is an instance of :class:`MagicAlias`, which holds the
name and kind of the magic it should call. Binding is done at
call time, so if the underlying magic function is changed the alias
will call the new function.
Parameters
----------
alias_name : str
The name of the magic to be registered.
magic_name : str
The name of an existing magic.
magic_kind : str
Kind of magic, one of 'line' or 'cell'
"""
# `validate_type` is too permissive, as it allows 'line_cell'
# which we do not handle.
if magic_kind not in magic_kinds:
raise ValueError('magic_kind must be one of %s, %s given' %
magic_kinds, magic_kind)
alias = MagicAlias(self.shell, magic_name, magic_kind, magic_params)
setattr(self.user_magics, alias_name, alias)
record_magic(self.magics, magic_kind, alias_name, alias)
# Key base class that provides the central functionality for magics.
class Magics(Configurable):
"""Base class for implementing magic functions.
Shell functions which can be reached as %function_name. All magic
functions should accept a string, which they can parse for their own
needs. This can make some functions easier to type, eg `%cd ../`
vs. `%cd("../")`
Classes providing magic functions need to subclass this class, and they
MUST:
- Use the method decorators `@line_magic` and `@cell_magic` to decorate
individual methods as magic functions, AND
- Use the class decorator `@magics_class` to ensure that the magic
methods are properly registered at the instance level upon instance
initialization.
See :mod:`magic_functions` for examples of actual implementation classes.
"""
# Dict holding all command-line options for each magic.
options_table = None
# Dict for the mapping of magic names to methods, set by class decorator
magics = None
# Flag to check that the class decorator was properly applied
registered = False
# Instance of IPython shell
shell = None
def __init__(self, shell=None, **kwargs):
if not(self.__class__.registered):
raise ValueError('Magics subclass without registration - '
'did you forget to apply @magics_class?')
if shell is not None:
if hasattr(shell, 'configurables'):
shell.configurables.append(self)
if hasattr(shell, 'config'):
kwargs.setdefault('parent', shell)
self.shell = shell
self.options_table = {}
# The method decorators are run when the instance doesn't exist yet, so
# they can only record the names of the methods they are supposed to
# grab. Only now, that the instance exists, can we create the proper
# mapping to bound methods. So we read the info off the original names
# table and replace each method name by the actual bound method.
# But we mustn't clobber the *class* mapping, in case of multiple instances.
class_magics = self.magics
self.magics = {}
for mtype in magic_kinds:
tab = self.magics[mtype] = {}
cls_tab = class_magics[mtype]
for magic_name, meth_name in cls_tab.items():
if isinstance(meth_name, str):
# it's a method name, grab it
tab[magic_name] = getattr(self, meth_name)
else:
# it's the real thing
tab[magic_name] = meth_name
# Configurable **needs** to be initiated at the end or the config
# magics get screwed up.
super(Magics, self).__init__(**kwargs)
def arg_err(self,func):
"""Print docstring if incorrect arguments were passed"""
print('Error in arguments:')
print(oinspect.getdoc(func))
def format_latex(self, strng):
"""Format a string for latex inclusion."""
# Characters that need to be escaped for latex:
escape_re = re.compile(r'(%|_|\$|#|&)',re.MULTILINE)
# Magic command names as headers:
cmd_name_re = re.compile(r'^(%s.*?):' % ESC_MAGIC,
re.MULTILINE)
# Magic commands
cmd_re = re.compile(r'(?P<cmd>%s.+?\b)(?!\}\}:)' % ESC_MAGIC,
re.MULTILINE)
# Paragraph continue
par_re = re.compile(r'\\$',re.MULTILINE)
# The "\n" symbol
newline_re = re.compile(r'\\n')
# Now build the string for output:
#strng = cmd_name_re.sub(r'\n\\texttt{\\textsl{\\large \1}}:',strng)
strng = cmd_name_re.sub(r'\n\\bigskip\n\\texttt{\\textbf{ \1}}:',
strng)
strng = cmd_re.sub(r'\\texttt{\g<cmd>}',strng)
strng = par_re.sub(r'\\\\',strng)
strng = escape_re.sub(r'\\\1',strng)
strng = newline_re.sub(r'\\textbackslash{}n',strng)
return strng
def parse_options(self, arg_str, opt_str, *long_opts, **kw):
"""Parse options passed to an argument string.
The interface is similar to that of :func:`getopt.getopt`, but it
returns a :class:`~IPython.utils.struct.Struct` with the options as keys
and the stripped argument string still as a string.
arg_str is quoted as a true sys.argv vector by using shlex.split.
This allows us to easily expand variables, glob files, quote
arguments, etc.
Parameters
----------
arg_str : str
The arguments to parse.
opt_str : str
The options specification.
mode : str, default 'string'
If given as 'list', the argument string is returned as a list (split
on whitespace) instead of a string.
list_all : bool, default False
Put all option values in lists. Normally only options
appearing more than once are put in a list.
posix : bool, default True
Whether to split the input line in POSIX mode or not, as per the
conventions outlined in the :mod:`shlex` module from the standard
library.
"""
# inject default options at the beginning of the input line
caller = sys._getframe(1).f_code.co_name
arg_str = '%s %s' % (self.options_table.get(caller,''),arg_str)
mode = kw.get('mode','string')
if mode not in ['string','list']:
raise ValueError('incorrect mode given: %s' % mode)
# Get options
list_all = kw.get('list_all',0)
posix = kw.get('posix', os.name == 'posix')
strict = kw.get('strict', True)
preserve_non_opts = kw.get("preserve_non_opts", False)
remainder_arg_str = arg_str
# Check if we have more than one argument to warrant extra processing:
odict = {} # Dictionary with options
args = arg_str.split()
if len(args) >= 1:
# If the list of inputs only has 0 or 1 thing in it, there's no
# need to look for options
argv = arg_split(arg_str, posix, strict)
# Do regular option processing
try:
opts,args = getopt(argv, opt_str, long_opts)
except GetoptError as e:
raise UsageError(
'%s ( allowed: "%s" %s)' % (e.msg, opt_str, " ".join(long_opts))
) from e
for o, a in opts:
if mode == "string" and preserve_non_opts:
# remove option-parts from the original args-string and preserve remaining-part.
# This relies on the arg_split(...) and getopt(...)'s impl spec, that the parsed options are
# returned in the original order.
remainder_arg_str = remainder_arg_str.replace(o, "", 1).replace(
a, "", 1
)
if o.startswith("--"):
o = o[2:]
else:
o = o[1:]
try:
odict[o].append(a)
except AttributeError:
odict[o] = [odict[o],a]
except KeyError:
if list_all:
odict[o] = [a]
else:
odict[o] = a
# Prepare opts,args for return
opts = Struct(odict)
if mode == 'string':
if preserve_non_opts:
args = remainder_arg_str.lstrip()
else:
args = " ".join(args)
return opts,args
def default_option(self, fn, optstr):
"""Make an entry in the options_table for fn, with value optstr"""
if fn not in self.lsmagic():
error("%s is not a magic function" % fn)
self.options_table[fn] = optstr
class MagicAlias(object):
"""An alias to another magic function.
An alias is determined by its magic name and magic kind. Lookup
is done at call time, so if the underlying magic changes the alias
will call the new function.
Use the :meth:`MagicsManager.register_alias` method or the
`%alias_magic` magic function to create and register a new alias.
"""
def __init__(self, shell, magic_name, magic_kind, magic_params=None):
self.shell = shell
self.magic_name = magic_name
self.magic_params = magic_params
self.magic_kind = magic_kind
self.pretty_target = '%s%s' % (magic_escapes[self.magic_kind], self.magic_name)
self.__doc__ = "Alias for `%s`." % self.pretty_target
self._in_call = False
def __call__(self, *args, **kwargs):
"""Call the magic alias."""
fn = self.shell.find_magic(self.magic_name, self.magic_kind)
if fn is None:
raise UsageError("Magic `%s` not found." % self.pretty_target)
# Protect against infinite recursion.
if self._in_call:
raise UsageError("Infinite recursion detected; "
"magic aliases cannot call themselves.")
self._in_call = True
try:
if self.magic_params:
args_list = list(args)
args_list[0] = self.magic_params + " " + args[0]
args = tuple(args_list)
return fn(*args, **kwargs)
finally:
self._in_call = False

@ -0,0 +1,310 @@
''' A decorator-based method of constructing IPython magics with `argparse`
option handling.
New magic functions can be defined like so::
from IPython.core.magic_arguments import (argument, magic_arguments,
parse_argstring)
@magic_arguments()
@argument('-o', '--option', help='An optional argument.')
@argument('arg', type=int, help='An integer positional argument.')
def magic_cool(self, arg):
""" A really cool magic command.
"""
args = parse_argstring(magic_cool, arg)
...
The `@magic_arguments` decorator marks the function as having argparse arguments.
The `@argument` decorator adds an argument using the same syntax as argparse's
`add_argument()` method. More sophisticated uses may also require the
`@argument_group` or `@kwds` decorator to customize the formatting and the
parsing.
Help text for the magic is automatically generated from the docstring and the
arguments::
In[1]: %cool?
%cool [-o OPTION] arg
A really cool magic command.
positional arguments:
arg An integer positional argument.
optional arguments:
-o OPTION, --option OPTION
An optional argument.
Here is an elaborated example that uses default parameters in `argument` and calls the `args` in the cell magic::
from IPython.core.magic import register_cell_magic
from IPython.core.magic_arguments import (argument, magic_arguments,
parse_argstring)
@magic_arguments()
@argument(
"--option",
"-o",
help=("Add an option here"),
)
@argument(
"--style",
"-s",
default="foo",
help=("Add some style arguments"),
)
@register_cell_magic
def my_cell_magic(line, cell):
args = parse_argstring(my_cell_magic, line)
print(f"{args.option=}")
print(f"{args.style=}")
print(f"{cell=}")
In a jupyter notebook, this cell magic can be executed like this::
%%my_cell_magic -o Hello
print("bar")
i = 42
Inheritance diagram:
.. inheritance-diagram:: IPython.core.magic_arguments
:parts: 3
'''
#-----------------------------------------------------------------------------
# Copyright (C) 2010-2011, IPython Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
import argparse
import re
# Our own imports
from IPython.core.error import UsageError
from IPython.utils.decorators import undoc
from IPython.utils.process import arg_split
from IPython.utils.text import dedent
NAME_RE = re.compile(r"[a-zA-Z][a-zA-Z0-9_-]*$")
@undoc
class MagicHelpFormatter(argparse.RawDescriptionHelpFormatter):
"""A HelpFormatter with a couple of changes to meet our needs.
"""
# Modified to dedent text.
def _fill_text(self, text, width, indent):
return argparse.RawDescriptionHelpFormatter._fill_text(self, dedent(text), width, indent)
# Modified to wrap argument placeholders in <> where necessary.
def _format_action_invocation(self, action):
if not action.option_strings:
metavar, = self._metavar_formatter(action, action.dest)(1)
return metavar
else:
parts = []
# if the Optional doesn't take a value, format is:
# -s, --long
if action.nargs == 0:
parts.extend(action.option_strings)
# if the Optional takes a value, format is:
# -s ARGS, --long ARGS
else:
default = action.dest.upper()
args_string = self._format_args(action, default)
# IPYTHON MODIFICATION: If args_string is not a plain name, wrap
# it in <> so it's valid RST.
if not NAME_RE.match(args_string):
args_string = "<%s>" % args_string
for option_string in action.option_strings:
parts.append('%s %s' % (option_string, args_string))
return ', '.join(parts)
# Override the default prefix ('usage') to our % magic escape,
# in a code block.
def add_usage(self, usage, actions, groups, prefix="::\n\n %"):
super(MagicHelpFormatter, self).add_usage(usage, actions, groups, prefix)
class MagicArgumentParser(argparse.ArgumentParser):
""" An ArgumentParser tweaked for use by IPython magics.
"""
def __init__(self,
prog=None,
usage=None,
description=None,
epilog=None,
parents=None,
formatter_class=MagicHelpFormatter,
prefix_chars='-',
argument_default=None,
conflict_handler='error',
add_help=False):
if parents is None:
parents = []
super(MagicArgumentParser, self).__init__(prog=prog, usage=usage,
description=description, epilog=epilog,
parents=parents, formatter_class=formatter_class,
prefix_chars=prefix_chars, argument_default=argument_default,
conflict_handler=conflict_handler, add_help=add_help)
def error(self, message):
""" Raise a catchable error instead of exiting.
"""
raise UsageError(message)
def parse_argstring(self, argstring):
""" Split a string into an argument list and parse that argument list.
"""
argv = arg_split(argstring)
return self.parse_args(argv)
def construct_parser(magic_func):
""" Construct an argument parser using the function decorations.
"""
kwds = getattr(magic_func, 'argcmd_kwds', {})
if 'description' not in kwds:
kwds['description'] = getattr(magic_func, '__doc__', None)
arg_name = real_name(magic_func)
parser = MagicArgumentParser(arg_name, **kwds)
# Reverse the list of decorators in order to apply them in the
# order in which they appear in the source.
group = None
for deco in magic_func.decorators[::-1]:
result = deco.add_to_parser(parser, group)
if result is not None:
group = result
# Replace the magic function's docstring with the full help text.
magic_func.__doc__ = parser.format_help()
return parser
def parse_argstring(magic_func, argstring):
""" Parse the string of arguments for the given magic function.
"""
return magic_func.parser.parse_argstring(argstring)
def real_name(magic_func):
""" Find the real name of the magic.
"""
magic_name = magic_func.__name__
if magic_name.startswith('magic_'):
magic_name = magic_name[len('magic_'):]
return getattr(magic_func, 'argcmd_name', magic_name)
class ArgDecorator(object):
""" Base class for decorators to add ArgumentParser information to a method.
"""
def __call__(self, func):
if not getattr(func, 'has_arguments', False):
func.has_arguments = True
func.decorators = []
func.decorators.append(self)
return func
def add_to_parser(self, parser, group):
""" Add this object's information to the parser, if necessary.
"""
pass
class magic_arguments(ArgDecorator):
""" Mark the magic as having argparse arguments and possibly adjust the
name.
"""
def __init__(self, name=None):
self.name = name
def __call__(self, func):
if not getattr(func, 'has_arguments', False):
func.has_arguments = True
func.decorators = []
if self.name is not None:
func.argcmd_name = self.name
# This should be the first decorator in the list of decorators, thus the
# last to execute. Build the parser.
func.parser = construct_parser(func)
return func
class ArgMethodWrapper(ArgDecorator):
"""
Base class to define a wrapper for ArgumentParser method.
Child class must define either `_method_name` or `add_to_parser`.
"""
_method_name = None
def __init__(self, *args, **kwds):
self.args = args
self.kwds = kwds
def add_to_parser(self, parser, group):
""" Add this object's information to the parser.
"""
if group is not None:
parser = group
getattr(parser, self._method_name)(*self.args, **self.kwds)
return None
class argument(ArgMethodWrapper):
""" Store arguments and keywords to pass to add_argument().
Instances also serve to decorate command methods.
"""
_method_name = 'add_argument'
class defaults(ArgMethodWrapper):
""" Store arguments and keywords to pass to set_defaults().
Instances also serve to decorate command methods.
"""
_method_name = 'set_defaults'
class argument_group(ArgMethodWrapper):
""" Store arguments and keywords to pass to add_argument_group().
Instances also serve to decorate command methods.
"""
def add_to_parser(self, parser, group):
""" Add this object's information to the parser.
"""
return parser.add_argument_group(*self.args, **self.kwds)
class kwds(ArgDecorator):
""" Provide other keywords to the sub-parser constructor.
"""
def __init__(self, **kwds):
self.kwds = kwds
def __call__(self, func):
func = super(kwds, self).__call__(func)
func.argcmd_kwds = self.kwds
return func
__all__ = ['magic_arguments', 'argument', 'argument_group', 'kwds',
'parse_argstring']

@ -0,0 +1,42 @@
"""Implementation of all the magic functions built into IPython.
"""
#-----------------------------------------------------------------------------
# Copyright (c) 2012 The IPython Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
from ..magic import Magics, magics_class
from .auto import AutoMagics
from .basic import BasicMagics, AsyncMagics
from .code import CodeMagics, MacroToEdit
from .config import ConfigMagics
from .display import DisplayMagics
from .execution import ExecutionMagics
from .extension import ExtensionMagics
from .history import HistoryMagics
from .logging import LoggingMagics
from .namespace import NamespaceMagics
from .osm import OSMagics
from .packaging import PackagingMagics
from .pylab import PylabMagics
from .script import ScriptMagics
#-----------------------------------------------------------------------------
# Magic implementation classes
#-----------------------------------------------------------------------------
@magics_class
class UserMagics(Magics):
"""Placeholder for user-defined magics to be added at runtime.
All magics are eventually merged into a single namespace at runtime, but we
use this class to isolate the magics defined dynamically by the user into
their own class.
"""

@ -0,0 +1,144 @@
"""Implementation of magic functions that control various automatic behaviors.
"""
#-----------------------------------------------------------------------------
# Copyright (c) 2012 The IPython Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
# Our own packages
from IPython.core.magic import Bunch, Magics, magics_class, line_magic
from IPython.testing.skipdoctest import skip_doctest
from logging import error
#-----------------------------------------------------------------------------
# Magic implementation classes
#-----------------------------------------------------------------------------
@magics_class
class AutoMagics(Magics):
"""Magics that control various autoX behaviors."""
def __init__(self, shell):
super(AutoMagics, self).__init__(shell)
# namespace for holding state we may need
self._magic_state = Bunch()
@line_magic
def automagic(self, parameter_s=''):
"""Make magic functions callable without having to type the initial %.
Without arguments toggles on/off (when off, you must call it as
%automagic, of course). With arguments it sets the value, and you can
use any of (case insensitive):
- on, 1, True: to activate
- off, 0, False: to deactivate.
Note that magic functions have lowest priority, so if there's a
variable whose name collides with that of a magic fn, automagic won't
work for that function (you get the variable instead). However, if you
delete the variable (del var), the previously shadowed magic function
becomes visible to automagic again."""
arg = parameter_s.lower()
mman = self.shell.magics_manager
if arg in ('on', '1', 'true'):
val = True
elif arg in ('off', '0', 'false'):
val = False
else:
val = not mman.auto_magic
mman.auto_magic = val
print('\n' + self.shell.magics_manager.auto_status())
@skip_doctest
@line_magic
def autocall(self, parameter_s=''):
"""Make functions callable without having to type parentheses.
Usage:
%autocall [mode]
The mode can be one of: 0->Off, 1->Smart, 2->Full. If not given, the
value is toggled on and off (remembering the previous state).
In more detail, these values mean:
0 -> fully disabled
1 -> active, but do not apply if there are no arguments on the line.
In this mode, you get::
In [1]: callable
Out[1]: <built-in function callable>
In [2]: callable 'hello'
------> callable('hello')
Out[2]: False
2 -> Active always. Even if no arguments are present, the callable
object is called::
In [2]: float
------> float()
Out[2]: 0.0
Note that even with autocall off, you can still use '/' at the start of
a line to treat the first argument on the command line as a function
and add parentheses to it::
In [8]: /str 43
------> str(43)
Out[8]: '43'
# all-random (note for auto-testing)
"""
valid_modes = {
0: "Off",
1: "Smart",
2: "Full",
}
def errorMessage() -> str:
error = "Valid modes: "
for k, v in valid_modes.items():
error += str(k) + "->" + v + ", "
error = error[:-2] # remove tailing `, ` after last element
return error
if parameter_s:
if not parameter_s in map(str, valid_modes.keys()):
error(errorMessage())
return
arg = int(parameter_s)
else:
arg = 'toggle'
if not arg in (*list(valid_modes.keys()), "toggle"):
error(errorMessage())
return
if arg in (valid_modes.keys()):
self.shell.autocall = arg
else: # toggle
if self.shell.autocall:
self._magic_state.autocall_save = self.shell.autocall
self.shell.autocall = 0
else:
try:
self.shell.autocall = self._magic_state.autocall_save
except AttributeError:
self.shell.autocall = self._magic_state.autocall_save = 1
print("Automatic calling is:", list(valid_modes.values())[self.shell.autocall])

@ -0,0 +1,659 @@
"""Implementation of basic magic functions."""
import argparse
from logging import error
import io
import os
from pprint import pformat
import sys
from warnings import warn
from traitlets.utils.importstring import import_item
from IPython.core import magic_arguments, page
from IPython.core.error import UsageError
from IPython.core.magic import Magics, magics_class, line_magic, magic_escapes
from IPython.utils.text import format_screen, dedent, indent
from IPython.testing.skipdoctest import skip_doctest
from IPython.utils.ipstruct import Struct
class MagicsDisplay(object):
def __init__(self, magics_manager, ignore=None):
self.ignore = ignore if ignore else []
self.magics_manager = magics_manager
def _lsmagic(self):
"""The main implementation of the %lsmagic"""
mesc = magic_escapes['line']
cesc = magic_escapes['cell']
mman = self.magics_manager
magics = mman.lsmagic()
out = ['Available line magics:',
mesc + (' '+mesc).join(sorted([m for m,v in magics['line'].items() if (v not in self.ignore)])),
'',
'Available cell magics:',
cesc + (' '+cesc).join(sorted([m for m,v in magics['cell'].items() if (v not in self.ignore)])),
'',
mman.auto_status()]
return '\n'.join(out)
def _repr_pretty_(self, p, cycle):
p.text(self._lsmagic())
def __str__(self):
return self._lsmagic()
def _jsonable(self):
"""turn magics dict into jsonable dict of the same structure
replaces object instances with their class names as strings
"""
magic_dict = {}
mman = self.magics_manager
magics = mman.lsmagic()
for key, subdict in magics.items():
d = {}
magic_dict[key] = d
for name, obj in subdict.items():
try:
classname = obj.__self__.__class__.__name__
except AttributeError:
classname = 'Other'
d[name] = classname
return magic_dict
def _repr_json_(self):
return self._jsonable()
@magics_class
class BasicMagics(Magics):
"""Magics that provide central IPython functionality.
These are various magics that don't fit into specific categories but that
are all part of the base 'IPython experience'."""
@skip_doctest
@magic_arguments.magic_arguments()
@magic_arguments.argument(
'-l', '--line', action='store_true',
help="""Create a line magic alias."""
)
@magic_arguments.argument(
'-c', '--cell', action='store_true',
help="""Create a cell magic alias."""
)
@magic_arguments.argument(
'name',
help="""Name of the magic to be created."""
)
@magic_arguments.argument(
'target',
help="""Name of the existing line or cell magic."""
)
@magic_arguments.argument(
'-p', '--params', default=None,
help="""Parameters passed to the magic function."""
)
@line_magic
def alias_magic(self, line=''):
"""Create an alias for an existing line or cell magic.
Examples
--------
::
In [1]: %alias_magic t timeit
Created `%t` as an alias for `%timeit`.
Created `%%t` as an alias for `%%timeit`.
In [2]: %t -n1 pass
1 loops, best of 3: 954 ns per loop
In [3]: %%t -n1
...: pass
...:
1 loops, best of 3: 954 ns per loop
In [4]: %alias_magic --cell whereami pwd
UsageError: Cell magic function `%%pwd` not found.
In [5]: %alias_magic --line whereami pwd
Created `%whereami` as an alias for `%pwd`.
In [6]: %whereami
Out[6]: u'/home/testuser'
In [7]: %alias_magic h history "-p -l 30" --line
Created `%h` as an alias for `%history -l 30`.
"""
args = magic_arguments.parse_argstring(self.alias_magic, line)
shell = self.shell
mman = self.shell.magics_manager
escs = ''.join(magic_escapes.values())
target = args.target.lstrip(escs)
name = args.name.lstrip(escs)
params = args.params
if (params and
((params.startswith('"') and params.endswith('"'))
or (params.startswith("'") and params.endswith("'")))):
params = params[1:-1]
# Find the requested magics.
m_line = shell.find_magic(target, 'line')
m_cell = shell.find_magic(target, 'cell')
if args.line and m_line is None:
raise UsageError('Line magic function `%s%s` not found.' %
(magic_escapes['line'], target))
if args.cell and m_cell is None:
raise UsageError('Cell magic function `%s%s` not found.' %
(magic_escapes['cell'], target))
# If --line and --cell are not specified, default to the ones
# that are available.
if not args.line and not args.cell:
if not m_line and not m_cell:
raise UsageError(
'No line or cell magic with name `%s` found.' % target
)
args.line = bool(m_line)
args.cell = bool(m_cell)
params_str = "" if params is None else " " + params
if args.line:
mman.register_alias(name, target, 'line', params)
print('Created `%s%s` as an alias for `%s%s%s`.' % (
magic_escapes['line'], name,
magic_escapes['line'], target, params_str))
if args.cell:
mman.register_alias(name, target, 'cell', params)
print('Created `%s%s` as an alias for `%s%s%s`.' % (
magic_escapes['cell'], name,
magic_escapes['cell'], target, params_str))
@line_magic
def lsmagic(self, parameter_s=''):
"""List currently available magic functions."""
return MagicsDisplay(self.shell.magics_manager, ignore=[])
def _magic_docs(self, brief=False, rest=False):
"""Return docstrings from magic functions."""
mman = self.shell.magics_manager
docs = mman.lsmagic_docs(brief, missing='No documentation')
if rest:
format_string = '**%s%s**::\n\n%s\n\n'
else:
format_string = '%s%s:\n%s\n'
return ''.join(
[format_string % (magic_escapes['line'], fname,
indent(dedent(fndoc)))
for fname, fndoc in sorted(docs['line'].items())]
+
[format_string % (magic_escapes['cell'], fname,
indent(dedent(fndoc)))
for fname, fndoc in sorted(docs['cell'].items())]
)
@line_magic
def magic(self, parameter_s=''):
"""Print information about the magic function system.
Supported formats: -latex, -brief, -rest
"""
mode = ''
try:
mode = parameter_s.split()[0][1:]
except IndexError:
pass
brief = (mode == 'brief')
rest = (mode == 'rest')
magic_docs = self._magic_docs(brief, rest)
if mode == 'latex':
print(self.format_latex(magic_docs))
return
else:
magic_docs = format_screen(magic_docs)
out = ["""
IPython's 'magic' functions
===========================
The magic function system provides a series of functions which allow you to
control the behavior of IPython itself, plus a lot of system-type
features. There are two kinds of magics, line-oriented and cell-oriented.
Line magics are prefixed with the % character and work much like OS
command-line calls: they get as an argument the rest of the line, where
arguments are passed without parentheses or quotes. For example, this will
time the given statement::
%timeit range(1000)
Cell magics are prefixed with a double %%, and they are functions that get as
an argument not only the rest of the line, but also the lines below it in a
separate argument. These magics are called with two arguments: the rest of the
call line and the body of the cell, consisting of the lines below the first.
For example::
%%timeit x = numpy.random.randn((100, 100))
numpy.linalg.svd(x)
will time the execution of the numpy svd routine, running the assignment of x
as part of the setup phase, which is not timed.
In a line-oriented client (the terminal or Qt console IPython), starting a new
input with %% will automatically enter cell mode, and IPython will continue
reading input until a blank line is given. In the notebook, simply type the
whole cell as one entity, but keep in mind that the %% escape can only be at
the very start of the cell.
NOTE: If you have 'automagic' enabled (via the command line option or with the
%automagic function), you don't need to type in the % explicitly for line
magics; cell magics always require an explicit '%%' escape. By default,
IPython ships with automagic on, so you should only rarely need the % escape.
Example: typing '%cd mydir' (without the quotes) changes your working directory
to 'mydir', if it exists.
For a list of the available magic functions, use %lsmagic. For a description
of any of them, type %magic_name?, e.g. '%cd?'.
Currently the magic system has the following functions:""",
magic_docs,
"Summary of magic functions (from %slsmagic):" % magic_escapes['line'],
str(self.lsmagic()),
]
page.page('\n'.join(out))
@line_magic
def page(self, parameter_s=''):
"""Pretty print the object and display it through a pager.
%page [options] OBJECT
If no object is given, use _ (last output).
Options:
-r: page str(object), don't pretty-print it."""
# After a function contributed by Olivier Aubert, slightly modified.
# Process options/args
opts, args = self.parse_options(parameter_s, 'r')
raw = 'r' in opts
oname = args and args or '_'
info = self.shell._ofind(oname)
if info['found']:
txt = (raw and str or pformat)( info['obj'] )
page.page(txt)
else:
print('Object `%s` not found' % oname)
@line_magic
def pprint(self, parameter_s=''):
"""Toggle pretty printing on/off."""
ptformatter = self.shell.display_formatter.formatters['text/plain']
ptformatter.pprint = bool(1 - ptformatter.pprint)
print('Pretty printing has been turned',
['OFF','ON'][ptformatter.pprint])
@line_magic
def colors(self, parameter_s=''):
"""Switch color scheme for prompts, info system and exception handlers.
Currently implemented schemes: NoColor, Linux, LightBG.
Color scheme names are not case-sensitive.
Examples
--------
To get a plain black and white terminal::
%colors nocolor
"""
def color_switch_err(name):
warn('Error changing %s color schemes.\n%s' %
(name, sys.exc_info()[1]), stacklevel=2)
new_scheme = parameter_s.strip()
if not new_scheme:
raise UsageError(
"%colors: you must specify a color scheme. See '%colors?'")
# local shortcut
shell = self.shell
# Set shell colour scheme
try:
shell.colors = new_scheme
shell.refresh_style()
except:
color_switch_err('shell')
# Set exception colors
try:
shell.InteractiveTB.set_colors(scheme = new_scheme)
shell.SyntaxTB.set_colors(scheme = new_scheme)
except:
color_switch_err('exception')
# Set info (for 'object?') colors
if shell.color_info:
try:
shell.inspector.set_active_scheme(new_scheme)
except:
color_switch_err('object inspector')
else:
shell.inspector.set_active_scheme('NoColor')
@line_magic
def xmode(self, parameter_s=''):
"""Switch modes for the exception handlers.
Valid modes: Plain, Context, Verbose, and Minimal.
If called without arguments, acts as a toggle.
When in verbose mode the value --show (and --hide)
will respectively show (or hide) frames with ``__tracebackhide__ =
True`` value set.
"""
def xmode_switch_err(name):
warn('Error changing %s exception modes.\n%s' %
(name,sys.exc_info()[1]))
shell = self.shell
if parameter_s.strip() == "--show":
shell.InteractiveTB.skip_hidden = False
return
if parameter_s.strip() == "--hide":
shell.InteractiveTB.skip_hidden = True
return
new_mode = parameter_s.strip().capitalize()
try:
shell.InteractiveTB.set_mode(mode=new_mode)
print('Exception reporting mode:',shell.InteractiveTB.mode)
except:
xmode_switch_err('user')
@line_magic
def quickref(self, arg):
""" Show a quick reference sheet """
from IPython.core.usage import quick_reference
qr = quick_reference + self._magic_docs(brief=True)
page.page(qr)
@line_magic
def doctest_mode(self, parameter_s=''):
"""Toggle doctest mode on and off.
This mode is intended to make IPython behave as much as possible like a
plain Python shell, from the perspective of how its prompts, exceptions
and output look. This makes it easy to copy and paste parts of a
session into doctests. It does so by:
- Changing the prompts to the classic ``>>>`` ones.
- Changing the exception reporting mode to 'Plain'.
- Disabling pretty-printing of output.
Note that IPython also supports the pasting of code snippets that have
leading '>>>' and '...' prompts in them. This means that you can paste
doctests from files or docstrings (even if they have leading
whitespace), and the code will execute correctly. You can then use
'%history -t' to see the translated history; this will give you the
input after removal of all the leading prompts and whitespace, which
can be pasted back into an editor.
With these features, you can switch into this mode easily whenever you
need to do testing and changes to doctests, without having to leave
your existing IPython session.
"""
# Shorthands
shell = self.shell
meta = shell.meta
disp_formatter = self.shell.display_formatter
ptformatter = disp_formatter.formatters['text/plain']
# dstore is a data store kept in the instance metadata bag to track any
# changes we make, so we can undo them later.
dstore = meta.setdefault('doctest_mode',Struct())
save_dstore = dstore.setdefault
# save a few values we'll need to recover later
mode = save_dstore('mode',False)
save_dstore('rc_pprint',ptformatter.pprint)
save_dstore('xmode',shell.InteractiveTB.mode)
save_dstore('rc_separate_out',shell.separate_out)
save_dstore('rc_separate_out2',shell.separate_out2)
save_dstore('rc_separate_in',shell.separate_in)
save_dstore('rc_active_types',disp_formatter.active_types)
if not mode:
# turn on
# Prompt separators like plain python
shell.separate_in = ''
shell.separate_out = ''
shell.separate_out2 = ''
ptformatter.pprint = False
disp_formatter.active_types = ['text/plain']
shell.magic('xmode Plain')
else:
# turn off
shell.separate_in = dstore.rc_separate_in
shell.separate_out = dstore.rc_separate_out
shell.separate_out2 = dstore.rc_separate_out2
ptformatter.pprint = dstore.rc_pprint
disp_formatter.active_types = dstore.rc_active_types
shell.magic('xmode ' + dstore.xmode)
# mode here is the state before we switch; switch_doctest_mode takes
# the mode we're switching to.
shell.switch_doctest_mode(not mode)
# Store new mode and inform
dstore.mode = bool(not mode)
mode_label = ['OFF','ON'][dstore.mode]
print('Doctest mode is:', mode_label)
@line_magic
def gui(self, parameter_s=''):
"""Enable or disable IPython GUI event loop integration.
%gui [GUINAME]
This magic replaces IPython's threaded shells that were activated
using the (pylab/wthread/etc.) command line flags. GUI toolkits
can now be enabled at runtime and keyboard
interrupts should work without any problems. The following toolkits
are supported: wxPython, PyQt4, PyGTK, Tk and Cocoa (OSX)::
%gui wx # enable wxPython event loop integration
%gui qt4|qt # enable PyQt4 event loop integration
%gui qt5 # enable PyQt5 event loop integration
%gui gtk # enable PyGTK event loop integration
%gui gtk3 # enable Gtk3 event loop integration
%gui gtk4 # enable Gtk4 event loop integration
%gui tk # enable Tk event loop integration
%gui osx # enable Cocoa event loop integration
# (requires %matplotlib 1.1)
%gui # disable all event loop integration
WARNING: after any of these has been called you can simply create
an application object, but DO NOT start the event loop yourself, as
we have already handled that.
"""
opts, arg = self.parse_options(parameter_s, '')
if arg=='': arg = None
try:
return self.shell.enable_gui(arg)
except Exception as e:
# print simple error message, rather than traceback if we can't
# hook up the GUI
error(str(e))
@skip_doctest
@line_magic
def precision(self, s=''):
"""Set floating point precision for pretty printing.
Can set either integer precision or a format string.
If numpy has been imported and precision is an int,
numpy display precision will also be set, via ``numpy.set_printoptions``.
If no argument is given, defaults will be restored.
Examples
--------
::
In [1]: from math import pi
In [2]: %precision 3
Out[2]: u'%.3f'
In [3]: pi
Out[3]: 3.142
In [4]: %precision %i
Out[4]: u'%i'
In [5]: pi
Out[5]: 3
In [6]: %precision %e
Out[6]: u'%e'
In [7]: pi**10
Out[7]: 9.364805e+04
In [8]: %precision
Out[8]: u'%r'
In [9]: pi**10
Out[9]: 93648.047476082982
"""
ptformatter = self.shell.display_formatter.formatters['text/plain']
ptformatter.float_precision = s
return ptformatter.float_format
@magic_arguments.magic_arguments()
@magic_arguments.argument(
'filename', type=str,
help='Notebook name or filename'
)
@line_magic
def notebook(self, s):
"""Export and convert IPython notebooks.
This function can export the current IPython history to a notebook file.
For example, to export the history to "foo.ipynb" do "%notebook foo.ipynb".
"""
args = magic_arguments.parse_argstring(self.notebook, s)
outfname = os.path.expanduser(args.filename)
from nbformat import write, v4
cells = []
hist = list(self.shell.history_manager.get_range())
if(len(hist)<=1):
raise ValueError('History is empty, cannot export')
for session, execution_count, source in hist[:-1]:
cells.append(v4.new_code_cell(
execution_count=execution_count,
source=source
))
nb = v4.new_notebook(cells=cells)
with io.open(outfname, "w", encoding="utf-8") as f:
write(nb, f, version=4)
@magics_class
class AsyncMagics(BasicMagics):
@line_magic
def autoawait(self, parameter_s):
"""
Allow to change the status of the autoawait option.
This allow you to set a specific asynchronous code runner.
If no value is passed, print the currently used asynchronous integration
and whether it is activated.
It can take a number of value evaluated in the following order:
- False/false/off deactivate autoawait integration
- True/true/on activate autoawait integration using configured default
loop
- asyncio/curio/trio activate autoawait integration and use integration
with said library.
- `sync` turn on the pseudo-sync integration (mostly used for
`IPython.embed()` which does not run IPython with a real eventloop and
deactivate running asynchronous code. Turning on Asynchronous code with
the pseudo sync loop is undefined behavior and may lead IPython to crash.
If the passed parameter does not match any of the above and is a python
identifier, get said object from user namespace and set it as the
runner, and activate autoawait.
If the object is a fully qualified object name, attempt to import it and
set it as the runner, and activate autoawait.
The exact behavior of autoawait is experimental and subject to change
across version of IPython and Python.
"""
param = parameter_s.strip()
d = {True: "on", False: "off"}
if not param:
print("IPython autoawait is `{}`, and set to use `{}`".format(
d[self.shell.autoawait],
self.shell.loop_runner
))
return None
if param.lower() in ('false', 'off'):
self.shell.autoawait = False
return None
if param.lower() in ('true', 'on'):
self.shell.autoawait = True
return None
if param in self.shell.loop_runner_map:
self.shell.loop_runner, self.shell.autoawait = self.shell.loop_runner_map[param]
return None
if param in self.shell.user_ns :
self.shell.loop_runner = self.shell.user_ns[param]
self.shell.autoawait = True
return None
runner = import_item(param)
self.shell.loop_runner = runner
self.shell.autoawait = True

@ -0,0 +1,755 @@
"""Implementation of code management magic functions.
"""
#-----------------------------------------------------------------------------
# Copyright (c) 2012 The IPython Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
# Stdlib
import inspect
import io
import os
import re
import sys
import ast
from itertools import chain
from urllib.request import Request, urlopen
from urllib.parse import urlencode
from pathlib import Path
# Our own packages
from IPython.core.error import TryNext, StdinNotImplementedError, UsageError
from IPython.core.macro import Macro
from IPython.core.magic import Magics, magics_class, line_magic
from IPython.core.oinspect import find_file, find_source_lines
from IPython.core.release import version
from IPython.testing.skipdoctest import skip_doctest
from IPython.utils.contexts import preserve_keys
from IPython.utils.path import get_py_filename
from warnings import warn
from logging import error
from IPython.utils.text import get_text_list
#-----------------------------------------------------------------------------
# Magic implementation classes
#-----------------------------------------------------------------------------
# Used for exception handling in magic_edit
class MacroToEdit(ValueError): pass
ipython_input_pat = re.compile(r"<ipython\-input\-(\d+)-[a-z\d]+>$")
# To match, e.g. 8-10 1:5 :10 3-
range_re = re.compile(r"""
(?P<start>\d+)?
((?P<sep>[\-:])
(?P<end>\d+)?)?
$""", re.VERBOSE)
def extract_code_ranges(ranges_str):
"""Turn a string of range for %%load into 2-tuples of (start, stop)
ready to use as a slice of the content split by lines.
Examples
--------
list(extract_input_ranges("5-10 2"))
[(4, 10), (1, 2)]
"""
for range_str in ranges_str.split():
rmatch = range_re.match(range_str)
if not rmatch:
continue
sep = rmatch.group("sep")
start = rmatch.group("start")
end = rmatch.group("end")
if sep == '-':
start = int(start) - 1 if start else None
end = int(end) if end else None
elif sep == ':':
start = int(start) - 1 if start else None
end = int(end) - 1 if end else None
else:
end = int(start)
start = int(start) - 1
yield (start, end)
def extract_symbols(code, symbols):
"""
Return a tuple (blocks, not_found)
where ``blocks`` is a list of code fragments
for each symbol parsed from code, and ``not_found`` are
symbols not found in the code.
For example::
In [1]: code = '''a = 10
...: def b(): return 42
...: class A: pass'''
In [2]: extract_symbols(code, 'A,b,z')
Out[2]: (['class A: pass\\n', 'def b(): return 42\\n'], ['z'])
"""
symbols = symbols.split(',')
# this will raise SyntaxError if code isn't valid Python
py_code = ast.parse(code)
marks = [(getattr(s, 'name', None), s.lineno) for s in py_code.body]
code = code.split('\n')
symbols_lines = {}
# we already know the start_lineno of each symbol (marks).
# To find each end_lineno, we traverse in reverse order until each
# non-blank line
end = len(code)
for name, start in reversed(marks):
while not code[end - 1].strip():
end -= 1
if name:
symbols_lines[name] = (start - 1, end)
end = start - 1
# Now symbols_lines is a map
# {'symbol_name': (start_lineno, end_lineno), ...}
# fill a list with chunks of codes for each requested symbol
blocks = []
not_found = []
for symbol in symbols:
if symbol in symbols_lines:
start, end = symbols_lines[symbol]
blocks.append('\n'.join(code[start:end]) + '\n')
else:
not_found.append(symbol)
return blocks, not_found
def strip_initial_indent(lines):
"""For %load, strip indent from lines until finding an unindented line.
https://github.com/ipython/ipython/issues/9775
"""
indent_re = re.compile(r'\s+')
it = iter(lines)
first_line = next(it)
indent_match = indent_re.match(first_line)
if indent_match:
# First line was indented
indent = indent_match.group()
yield first_line[len(indent):]
for line in it:
if line.startswith(indent):
yield line[len(indent):]
else:
# Less indented than the first line - stop dedenting
yield line
break
else:
yield first_line
# Pass the remaining lines through without dedenting
for line in it:
yield line
class InteractivelyDefined(Exception):
"""Exception for interactively defined variable in magic_edit"""
def __init__(self, index):
self.index = index
@magics_class
class CodeMagics(Magics):
"""Magics related to code management (loading, saving, editing, ...)."""
def __init__(self, *args, **kwargs):
self._knowntemps = set()
super(CodeMagics, self).__init__(*args, **kwargs)
@line_magic
def save(self, parameter_s=''):
"""Save a set of lines or a macro to a given filename.
Usage:\\
%save [options] filename [history]
Options:
-r: use 'raw' input. By default, the 'processed' history is used,
so that magics are loaded in their transformed version to valid
Python. If this option is given, the raw input as typed as the
command line is used instead.
-f: force overwrite. If file exists, %save will prompt for overwrite
unless -f is given.
-a: append to the file instead of overwriting it.
The history argument uses the same syntax as %history for input ranges,
then saves the lines to the filename you specify.
If no ranges are specified, saves history of the current session up to
this point.
It adds a '.py' extension to the file if you don't do so yourself, and
it asks for confirmation before overwriting existing files.
If `-r` option is used, the default extension is `.ipy`.
"""
opts,args = self.parse_options(parameter_s,'fra',mode='list')
if not args:
raise UsageError('Missing filename.')
raw = 'r' in opts
force = 'f' in opts
append = 'a' in opts
mode = 'a' if append else 'w'
ext = '.ipy' if raw else '.py'
fname, codefrom = args[0], " ".join(args[1:])
if not fname.endswith(('.py','.ipy')):
fname += ext
fname = os.path.expanduser(fname)
file_exists = os.path.isfile(fname)
if file_exists and not force and not append:
try:
overwrite = self.shell.ask_yes_no('File `%s` exists. Overwrite (y/[N])? ' % fname, default='n')
except StdinNotImplementedError:
print("File `%s` exists. Use `%%save -f %s` to force overwrite" % (fname, parameter_s))
return
if not overwrite :
print('Operation cancelled.')
return
try:
cmds = self.shell.find_user_code(codefrom,raw)
except (TypeError, ValueError) as e:
print(e.args[0])
return
with io.open(fname, mode, encoding="utf-8") as f:
if not file_exists or not append:
f.write("# coding: utf-8\n")
f.write(cmds)
# make sure we end on a newline
if not cmds.endswith('\n'):
f.write('\n')
print('The following commands were written to file `%s`:' % fname)
print(cmds)
@line_magic
def pastebin(self, parameter_s=''):
"""Upload code to dpaste.com, returning the URL.
Usage:\\
%pastebin [-d "Custom description"][-e 24] 1-7
The argument can be an input history range, a filename, or the name of a
string or macro.
If no arguments are given, uploads the history of this session up to
this point.
Options:
-d: Pass a custom description. The default will say
"Pasted from IPython".
-e: Pass number of days for the link to be expired.
The default will be 7 days.
"""
opts, args = self.parse_options(parameter_s, "d:e:")
try:
code = self.shell.find_user_code(args)
except (ValueError, TypeError) as e:
print(e.args[0])
return
expiry_days = 7
try:
expiry_days = int(opts.get("e", 7))
except ValueError as e:
print(e.args[0].capitalize())
return
if expiry_days < 1 or expiry_days > 365:
print("Expiry days should be in range of 1 to 365")
return
post_data = urlencode(
{
"title": opts.get("d", "Pasted from IPython"),
"syntax": "python",
"content": code,
"expiry_days": expiry_days,
}
).encode("utf-8")
request = Request(
"https://dpaste.com/api/v2/",
headers={"User-Agent": "IPython v{}".format(version)},
)
response = urlopen(request, post_data)
return response.headers.get('Location')
@line_magic
def loadpy(self, arg_s):
"""Alias of `%load`
`%loadpy` has gained some flexibility and dropped the requirement of a `.py`
extension. So it has been renamed simply into %load. You can look at
`%load`'s docstring for more info.
"""
self.load(arg_s)
@line_magic
def load(self, arg_s):
"""Load code into the current frontend.
Usage:\\
%load [options] source
where source can be a filename, URL, input history range, macro, or
element in the user namespace
If no arguments are given, loads the history of this session up to this
point.
Options:
-r <lines>: Specify lines or ranges of lines to load from the source.
Ranges could be specified as x-y (x..y) or in python-style x:y
(x..(y-1)). Both limits x and y can be left blank (meaning the
beginning and end of the file, respectively).
-s <symbols>: Specify function or classes to load from python source.
-y : Don't ask confirmation for loading source above 200 000 characters.
-n : Include the user's namespace when searching for source code.
This magic command can either take a local filename, a URL, an history
range (see %history) or a macro as argument, it will prompt for
confirmation before loading source with more than 200 000 characters, unless
-y flag is passed or if the frontend does not support raw_input::
%load
%load myscript.py
%load 7-27
%load myMacro
%load http://www.example.com/myscript.py
%load -r 5-10 myscript.py
%load -r 10-20,30,40: foo.py
%load -s MyClass,wonder_function myscript.py
%load -n MyClass
%load -n my_module.wonder_function
"""
opts,args = self.parse_options(arg_s,'yns:r:')
search_ns = 'n' in opts
contents = self.shell.find_user_code(args, search_ns=search_ns)
if 's' in opts:
try:
blocks, not_found = extract_symbols(contents, opts['s'])
except SyntaxError:
# non python code
error("Unable to parse the input as valid Python code")
return
if len(not_found) == 1:
warn('The symbol `%s` was not found' % not_found[0])
elif len(not_found) > 1:
warn('The symbols %s were not found' % get_text_list(not_found,
wrap_item_with='`')
)
contents = '\n'.join(blocks)
if 'r' in opts:
ranges = opts['r'].replace(',', ' ')
lines = contents.split('\n')
slices = extract_code_ranges(ranges)
contents = [lines[slice(*slc)] for slc in slices]
contents = '\n'.join(strip_initial_indent(chain.from_iterable(contents)))
l = len(contents)
# 200 000 is ~ 2500 full 80 character lines
# so in average, more than 5000 lines
if l > 200000 and 'y' not in opts:
try:
ans = self.shell.ask_yes_no(("The text you're trying to load seems pretty big"\
" (%d characters). Continue (y/[N]) ?" % l), default='n' )
except StdinNotImplementedError:
#assume yes if raw input not implemented
ans = True
if ans is False :
print('Operation cancelled.')
return
contents = "# %load {}\n".format(arg_s) + contents
self.shell.set_next_input(contents, replace=True)
@staticmethod
def _find_edit_target(shell, args, opts, last_call):
"""Utility method used by magic_edit to find what to edit."""
def make_filename(arg):
"Make a filename from the given args"
try:
filename = get_py_filename(arg)
except IOError:
# If it ends with .py but doesn't already exist, assume we want
# a new file.
if arg.endswith('.py'):
filename = arg
else:
filename = None
return filename
# Set a few locals from the options for convenience:
opts_prev = 'p' in opts
opts_raw = 'r' in opts
# custom exceptions
class DataIsObject(Exception): pass
# Default line number value
lineno = opts.get('n',None)
if opts_prev:
args = '_%s' % last_call[0]
if args not in shell.user_ns:
args = last_call[1]
# by default this is done with temp files, except when the given
# arg is a filename
use_temp = True
data = ''
# First, see if the arguments should be a filename.
filename = make_filename(args)
if filename:
use_temp = False
elif args:
# Mode where user specifies ranges of lines, like in %macro.
data = shell.extract_input_lines(args, opts_raw)
if not data:
try:
# Load the parameter given as a variable. If not a string,
# process it as an object instead (below)
#print '*** args',args,'type',type(args) # dbg
data = eval(args, shell.user_ns)
if not isinstance(data, str):
raise DataIsObject
except (NameError,SyntaxError):
# given argument is not a variable, try as a filename
filename = make_filename(args)
if filename is None:
warn("Argument given (%s) can't be found as a variable "
"or as a filename." % args)
return (None, None, None)
use_temp = False
except DataIsObject as e:
# macros have a special edit function
if isinstance(data, Macro):
raise MacroToEdit(data) from e
# For objects, try to edit the file where they are defined
filename = find_file(data)
if filename:
if 'fakemodule' in filename.lower() and \
inspect.isclass(data):
# class created by %edit? Try to find source
# by looking for method definitions instead, the
# __module__ in those classes is FakeModule.
attrs = [getattr(data, aname) for aname in dir(data)]
for attr in attrs:
if not inspect.ismethod(attr):
continue
filename = find_file(attr)
if filename and \
'fakemodule' not in filename.lower():
# change the attribute to be the edit
# target instead
data = attr
break
m = ipython_input_pat.match(os.path.basename(filename))
if m:
raise InteractivelyDefined(int(m.groups()[0])) from e
datafile = 1
if filename is None:
filename = make_filename(args)
datafile = 1
if filename is not None:
# only warn about this if we get a real name
warn('Could not find file where `%s` is defined.\n'
'Opening a file named `%s`' % (args, filename))
# Now, make sure we can actually read the source (if it was
# in a temp file it's gone by now).
if datafile:
if lineno is None:
lineno = find_source_lines(data)
if lineno is None:
filename = make_filename(args)
if filename is None:
warn('The file where `%s` was defined '
'cannot be read or found.' % data)
return (None, None, None)
use_temp = False
if use_temp:
filename = shell.mktempfile(data)
print('IPython will make a temporary file named:',filename)
# use last_call to remember the state of the previous call, but don't
# let it be clobbered by successive '-p' calls.
try:
last_call[0] = shell.displayhook.prompt_count
if not opts_prev:
last_call[1] = args
except:
pass
return filename, lineno, use_temp
def _edit_macro(self,mname,macro):
"""open an editor with the macro data in a file"""
filename = self.shell.mktempfile(macro.value)
self.shell.hooks.editor(filename)
# and make a new macro object, to replace the old one
mvalue = Path(filename).read_text(encoding="utf-8")
self.shell.user_ns[mname] = Macro(mvalue)
@skip_doctest
@line_magic
def edit(self, parameter_s='',last_call=['','']):
"""Bring up an editor and execute the resulting code.
Usage:
%edit [options] [args]
%edit runs IPython's editor hook. The default version of this hook is
set to call the editor specified by your $EDITOR environment variable.
If this isn't found, it will default to vi under Linux/Unix and to
notepad under Windows. See the end of this docstring for how to change
the editor hook.
You can also set the value of this editor via the
``TerminalInteractiveShell.editor`` option in your configuration file.
This is useful if you wish to use a different editor from your typical
default with IPython (and for Windows users who typically don't set
environment variables).
This command allows you to conveniently edit multi-line code right in
your IPython session.
If called without arguments, %edit opens up an empty editor with a
temporary file and will execute the contents of this file when you
close it (don't forget to save it!).
Options:
-n <number>: open the editor at a specified line number. By default,
the IPython editor hook uses the unix syntax 'editor +N filename', but
you can configure this by providing your own modified hook if your
favorite editor supports line-number specifications with a different
syntax.
-p: this will call the editor with the same data as the previous time
it was used, regardless of how long ago (in your current session) it
was.
-r: use 'raw' input. This option only applies to input taken from the
user's history. By default, the 'processed' history is used, so that
magics are loaded in their transformed version to valid Python. If
this option is given, the raw input as typed as the command line is
used instead. When you exit the editor, it will be executed by
IPython's own processor.
-x: do not execute the edited code immediately upon exit. This is
mainly useful if you are editing programs which need to be called with
command line arguments, which you can then do using %run.
Arguments:
If arguments are given, the following possibilities exist:
- If the argument is a filename, IPython will load that into the
editor. It will execute its contents with execfile() when you exit,
loading any code in the file into your interactive namespace.
- The arguments are ranges of input history, e.g. "7 ~1/4-6".
The syntax is the same as in the %history magic.
- If the argument is a string variable, its contents are loaded
into the editor. You can thus edit any string which contains
python code (including the result of previous edits).
- If the argument is the name of an object (other than a string),
IPython will try to locate the file where it was defined and open the
editor at the point where it is defined. You can use `%edit function`
to load an editor exactly at the point where 'function' is defined,
edit it and have the file be executed automatically.
- If the object is a macro (see %macro for details), this opens up your
specified editor with a temporary file containing the macro's data.
Upon exit, the macro is reloaded with the contents of the file.
Note: opening at an exact line is only supported under Unix, and some
editors (like kedit and gedit up to Gnome 2.8) do not understand the
'+NUMBER' parameter necessary for this feature. Good editors like
(X)Emacs, vi, jed, pico and joe all do.
After executing your code, %edit will return as output the code you
typed in the editor (except when it was an existing file). This way
you can reload the code in further invocations of %edit as a variable,
via _<NUMBER> or Out[<NUMBER>], where <NUMBER> is the prompt number of
the output.
Note that %edit is also available through the alias %ed.
This is an example of creating a simple function inside the editor and
then modifying it. First, start up the editor::
In [1]: edit
Editing... done. Executing edited code...
Out[1]: 'def foo():\\n print "foo() was defined in an editing
session"\\n'
We can then call the function foo()::
In [2]: foo()
foo() was defined in an editing session
Now we edit foo. IPython automatically loads the editor with the
(temporary) file where foo() was previously defined::
In [3]: edit foo
Editing... done. Executing edited code...
And if we call foo() again we get the modified version::
In [4]: foo()
foo() has now been changed!
Here is an example of how to edit a code snippet successive
times. First we call the editor::
In [5]: edit
Editing... done. Executing edited code...
hello
Out[5]: "print 'hello'\\n"
Now we call it again with the previous output (stored in _)::
In [6]: edit _
Editing... done. Executing edited code...
hello world
Out[6]: "print 'hello world'\\n"
Now we call it with the output #8 (stored in _8, also as Out[8])::
In [7]: edit _8
Editing... done. Executing edited code...
hello again
Out[7]: "print 'hello again'\\n"
Changing the default editor hook:
If you wish to write your own editor hook, you can put it in a
configuration file which you load at startup time. The default hook
is defined in the IPython.core.hooks module, and you can use that as a
starting example for further modifications. That file also has
general instructions on how to set a new hook for use once you've
defined it."""
opts,args = self.parse_options(parameter_s,'prxn:')
try:
filename, lineno, is_temp = self._find_edit_target(self.shell,
args, opts, last_call)
except MacroToEdit as e:
self._edit_macro(args, e.args[0])
return
except InteractivelyDefined as e:
print("Editing In[%i]" % e.index)
args = str(e.index)
filename, lineno, is_temp = self._find_edit_target(self.shell,
args, opts, last_call)
if filename is None:
# nothing was found, warnings have already been issued,
# just give up.
return
if is_temp:
self._knowntemps.add(filename)
elif (filename in self._knowntemps):
is_temp = True
# do actual editing here
print('Editing...', end=' ')
sys.stdout.flush()
filepath = Path(filename)
try:
# Quote filenames that may have spaces in them when opening
# the editor
quoted = filename = str(filepath.absolute())
if " " in quoted:
quoted = "'%s'" % quoted
self.shell.hooks.editor(quoted, lineno)
except TryNext:
warn('Could not open editor')
return
# XXX TODO: should this be generalized for all string vars?
# For now, this is special-cased to blocks created by cpaste
if args.strip() == "pasted_block":
self.shell.user_ns["pasted_block"] = filepath.read_text(encoding="utf-8")
if 'x' in opts: # -x prevents actual execution
print()
else:
print('done. Executing edited code...')
with preserve_keys(self.shell.user_ns, '__file__'):
if not is_temp:
self.shell.user_ns["__file__"] = filename
if "r" in opts: # Untranslated IPython code
source = filepath.read_text(encoding="utf-8")
self.shell.run_cell(source, store_history=False)
else:
self.shell.safe_execfile(filename, self.shell.user_ns,
self.shell.user_ns)
if is_temp:
try:
return filepath.read_text(encoding="utf-8")
except IOError as msg:
if Path(msg.filename) == filepath:
warn('File not found. Did you forget to save?')
return
else:
self.shell.showtraceback()

@ -0,0 +1,187 @@
"""Implementation of configuration-related magic functions.
"""
#-----------------------------------------------------------------------------
# Copyright (c) 2012 The IPython Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
# Stdlib
import re
# Our own packages
from IPython.core.error import UsageError
from IPython.core.magic import Magics, magics_class, line_magic
from logging import error
#-----------------------------------------------------------------------------
# Magic implementation classes
#-----------------------------------------------------------------------------
reg = re.compile(r'^\w+\.\w+$')
@magics_class
class ConfigMagics(Magics):
def __init__(self, shell):
super(ConfigMagics, self).__init__(shell)
self.configurables = []
@line_magic
def config(self, s):
"""configure IPython
%config Class[.trait=value]
This magic exposes most of the IPython config system. Any
Configurable class should be able to be configured with the simple
line::
%config Class.trait=value
Where `value` will be resolved in the user's namespace, if it is an
expression or variable name.
Examples
--------
To see what classes are available for config, pass no arguments::
In [1]: %config
Available objects for config:
AliasManager
DisplayFormatter
HistoryManager
IPCompleter
LoggingMagics
MagicsManager
OSMagics
PrefilterManager
ScriptMagics
TerminalInteractiveShell
To view what is configurable on a given class, just pass the class
name::
In [2]: %config IPCompleter
IPCompleter(Completer) options
----------------------------
IPCompleter.backslash_combining_completions=<Bool>
Enable unicode completions, e.g. \\alpha<tab> . Includes completion of latex
commands, unicode names, and expanding unicode characters back to latex
commands.
Current: True
IPCompleter.debug=<Bool>
Enable debug for the Completer. Mostly print extra information for
experimental jedi integration.
Current: False
IPCompleter.greedy=<Bool>
Activate greedy completion
PENDING DEPRECATION. this is now mostly taken care of with Jedi.
This will enable completion on elements of lists, results of function calls, etc.,
but can be unsafe because the code is actually evaluated on TAB.
Current: False
IPCompleter.jedi_compute_type_timeout=<Int>
Experimental: restrict time (in milliseconds) during which Jedi can compute types.
Set to 0 to stop computing types. Non-zero value lower than 100ms may hurt
performance by preventing jedi to build its cache.
Current: 400
IPCompleter.limit_to__all__=<Bool>
DEPRECATED as of version 5.0.
Instruct the completer to use __all__ for the completion
Specifically, when completing on ``object.<tab>``.
When True: only those names in obj.__all__ will be included.
When False [default]: the __all__ attribute is ignored
Current: False
IPCompleter.merge_completions=<Bool>
Whether to merge completion results into a single list
If False, only the completion results from the first non-empty
completer will be returned.
Current: True
IPCompleter.omit__names=<Enum>
Instruct the completer to omit private method names
Specifically, when completing on ``object.<tab>``.
When 2 [default]: all names that start with '_' will be excluded.
When 1: all 'magic' names (``__foo__``) will be excluded.
When 0: nothing will be excluded.
Choices: any of [0, 1, 2]
Current: 2
IPCompleter.profile_completions=<Bool>
If True, emit profiling data for completion subsystem using cProfile.
Current: False
IPCompleter.profiler_output_dir=<Unicode>
Template for path at which to output profile data for completions.
Current: '.completion_profiles'
IPCompleter.use_jedi=<Bool>
Experimental: Use Jedi to generate autocompletions. Default to True if jedi
is installed.
Current: True
but the real use is in setting values::
In [3]: %config IPCompleter.greedy = True
and these values are read from the user_ns if they are variables::
In [4]: feeling_greedy=False
In [5]: %config IPCompleter.greedy = feeling_greedy
"""
from traitlets.config.loader import Config
# some IPython objects are Configurable, but do not yet have
# any configurable traits. Exclude them from the effects of
# this magic, as their presence is just noise:
configurables = sorted(set([ c for c in self.shell.configurables
if c.__class__.class_traits(config=True)
]), key=lambda x: x.__class__.__name__)
classnames = [ c.__class__.__name__ for c in configurables ]
line = s.strip()
if not line:
# print available configurable names
print("Available objects for config:")
for name in classnames:
print(" ", name)
return
elif line in classnames:
# `%config TerminalInteractiveShell` will print trait info for
# TerminalInteractiveShell
c = configurables[classnames.index(line)]
cls = c.__class__
help = cls.class_get_help(c)
# strip leading '--' from cl-args:
help = re.sub(re.compile(r'^--', re.MULTILINE), '', help)
print(help)
return
elif reg.match(line):
cls, attr = line.split('.')
return getattr(configurables[classnames.index(cls)],attr)
elif '=' not in line:
msg = "Invalid config statement: %r, "\
"should be `Class.trait = value`."
ll = line.lower()
for classname in classnames:
if ll == classname.lower():
msg = msg + '\nDid you mean %s (note the case)?' % classname
break
raise UsageError( msg % line)
# otherwise, assume we are setting configurables.
# leave quotes on args when splitting, because we want
# unquoted args to eval in user_ns
cfg = Config()
exec("cfg."+line, self.shell.user_ns, locals())
for configurable in configurables:
try:
configurable.update_config(cfg)
except Exception as e:
error(e)

@ -0,0 +1,93 @@
"""Simple magics for display formats"""
#-----------------------------------------------------------------------------
# Copyright (c) 2012 The IPython Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
# Our own packages
from IPython.display import display, Javascript, Latex, SVG, HTML, Markdown
from IPython.core.magic import (
Magics, magics_class, cell_magic
)
from IPython.core import magic_arguments
#-----------------------------------------------------------------------------
# Magic implementation classes
#-----------------------------------------------------------------------------
@magics_class
class DisplayMagics(Magics):
"""Magics for displaying various output types with literals
Defines javascript/latex/svg/html cell magics for writing
blocks in those languages, to be rendered in the frontend.
"""
@cell_magic
def js(self, line, cell):
"""Run the cell block of Javascript code
Alias of `%%javascript`
Starting with IPython 8.0 %%javascript is pending deprecation to be replaced
by a more flexible system
Please See https://github.com/ipython/ipython/issues/13376
"""
self.javascript(line, cell)
@cell_magic
def javascript(self, line, cell):
"""Run the cell block of Javascript code
Starting with IPython 8.0 %%javascript is pending deprecation to be replaced
by a more flexible system
Please See https://github.com/ipython/ipython/issues/13376
"""
display(Javascript(cell))
@cell_magic
def latex(self, line, cell):
"""Render the cell as a block of LaTeX
The subset of LaTeX which is supported depends on the implementation in
the client. In the Jupyter Notebook, this magic only renders the subset
of LaTeX defined by MathJax
[here](https://docs.mathjax.org/en/v2.5-latest/tex.html)."""
display(Latex(cell))
@cell_magic
def svg(self, line, cell):
"""Render the cell as an SVG literal"""
display(SVG(cell))
@magic_arguments.magic_arguments()
@magic_arguments.argument(
'--isolated', action='store_true', default=False,
help="""Annotate the cell as 'isolated'.
Isolated cells are rendered inside their own <iframe> tag"""
)
@cell_magic
def html(self, line, cell):
"""Render the cell as a block of HTML"""
args = magic_arguments.parse_argstring(self.html, line)
html = HTML(cell)
if args.isolated:
display(html, metadata={'text/html':{'isolated':True}})
else:
display(html)
@cell_magic
def markdown(self, line, cell):
"""Render the cell as Markdown text block"""
display(Markdown(cell))

@ -0,0 +1,63 @@
"""Implementation of magic functions for the extension machinery.
"""
#-----------------------------------------------------------------------------
# Copyright (c) 2012 The IPython Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
# Our own packages
from IPython.core.error import UsageError
from IPython.core.magic import Magics, magics_class, line_magic
#-----------------------------------------------------------------------------
# Magic implementation classes
#-----------------------------------------------------------------------------
@magics_class
class ExtensionMagics(Magics):
"""Magics to manage the IPython extensions system."""
@line_magic
def load_ext(self, module_str):
"""Load an IPython extension by its module name."""
if not module_str:
raise UsageError('Missing module name.')
res = self.shell.extension_manager.load_extension(module_str)
if res == 'already loaded':
print("The %s extension is already loaded. To reload it, use:" % module_str)
print(" %reload_ext", module_str)
elif res == 'no load function':
print("The %s module is not an IPython extension." % module_str)
@line_magic
def unload_ext(self, module_str):
"""Unload an IPython extension by its module name.
Not all extensions can be unloaded, only those which define an
``unload_ipython_extension`` function.
"""
if not module_str:
raise UsageError('Missing module name.')
res = self.shell.extension_manager.unload_extension(module_str)
if res == 'no unload function':
print("The %s extension doesn't define how to unload it." % module_str)
elif res == "not loaded":
print("The %s extension is not loaded." % module_str)
@line_magic
def reload_ext(self, module_str):
"""Reload an IPython extension by its module name."""
if not module_str:
raise UsageError('Missing module name.')
self.shell.extension_manager.reload_extension(module_str)

@ -0,0 +1,338 @@
"""Implementation of magic functions related to History.
"""
#-----------------------------------------------------------------------------
# Copyright (c) 2012, IPython Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
# Stdlib
import os
import sys
from io import open as io_open
import fnmatch
# Our own packages
from IPython.core.error import StdinNotImplementedError
from IPython.core.magic import Magics, magics_class, line_magic
from IPython.core.magic_arguments import (argument, magic_arguments,
parse_argstring)
from IPython.testing.skipdoctest import skip_doctest
from IPython.utils import io
#-----------------------------------------------------------------------------
# Magics class implementation
#-----------------------------------------------------------------------------
_unspecified = object()
@magics_class
class HistoryMagics(Magics):
@magic_arguments()
@argument(
'-n', dest='print_nums', action='store_true', default=False,
help="""
print line numbers for each input.
This feature is only available if numbered prompts are in use.
""")
@argument(
'-o', dest='get_output', action='store_true', default=False,
help="also print outputs for each input.")
@argument(
'-p', dest='pyprompts', action='store_true', default=False,
help="""
print classic '>>>' python prompts before each input.
This is useful for making documentation, and in conjunction
with -o, for producing doctest-ready output.
""")
@argument(
'-t', dest='raw', action='store_false', default=True,
help="""
print the 'translated' history, as IPython understands it.
IPython filters your input and converts it all into valid Python
source before executing it (things like magics or aliases are turned
into function calls, for example). With this option, you'll see the
native history instead of the user-entered version: '%%cd /' will be
seen as 'get_ipython().run_line_magic("cd", "/")' instead of '%%cd /'.
""")
@argument(
'-f', dest='filename',
help="""
FILENAME: instead of printing the output to the screen, redirect
it to the given file. The file is always overwritten, though *when
it can*, IPython asks for confirmation first. In particular, running
the command 'history -f FILENAME' from the IPython Notebook
interface will replace FILENAME even if it already exists *without*
confirmation.
""")
@argument(
'-g', dest='pattern', nargs='*', default=None,
help="""
treat the arg as a glob pattern to search for in (full) history.
This includes the saved history (almost all commands ever written).
The pattern may contain '?' to match one unknown character and '*'
to match any number of unknown characters. Use '%%hist -g' to show
full saved history (may be very long).
""")
@argument(
'-l', dest='limit', type=int, nargs='?', default=_unspecified,
help="""
get the last n lines from all sessions. Specify n as a single
arg, or the default is the last 10 lines.
""")
@argument(
'-u', dest='unique', action='store_true',
help="""
when searching history using `-g`, show only unique history.
""")
@argument('range', nargs='*')
@skip_doctest
@line_magic
def history(self, parameter_s = ''):
"""Print input history (_i<n> variables), with most recent last.
By default, input history is printed without line numbers so it can be
directly pasted into an editor. Use -n to show them.
By default, all input history from the current session is displayed.
Ranges of history can be indicated using the syntax:
``4``
Line 4, current session
``4-6``
Lines 4-6, current session
``243/1-5``
Lines 1-5, session 243
``~2/7``
Line 7, session 2 before current
``~8/1-~6/5``
From the first line of 8 sessions ago, to the fifth line of 6
sessions ago.
Multiple ranges can be entered, separated by spaces
The same syntax is used by %macro, %save, %edit, %rerun
Examples
--------
::
In [6]: %history -n 4-6
4:a = 12
5:print a**2
6:%history -n 4-6
"""
args = parse_argstring(self.history, parameter_s)
# For brevity
history_manager = self.shell.history_manager
def _format_lineno(session, line):
"""Helper function to format line numbers properly."""
if session in (0, history_manager.session_number):
return str(line)
return "%s/%s" % (session, line)
# Check if output to specific file was requested.
outfname = args.filename
if not outfname:
outfile = sys.stdout # default
# We don't want to close stdout at the end!
close_at_end = False
else:
outfname = os.path.expanduser(outfname)
if os.path.exists(outfname):
try:
ans = io.ask_yes_no("File %r exists. Overwrite?" % outfname)
except StdinNotImplementedError:
ans = True
if not ans:
print('Aborting.')
return
print("Overwriting file.")
outfile = io_open(outfname, 'w', encoding='utf-8')
close_at_end = True
print_nums = args.print_nums
get_output = args.get_output
pyprompts = args.pyprompts
raw = args.raw
pattern = None
limit = None if args.limit is _unspecified else args.limit
range_pattern = False
if args.pattern is not None and not args.range:
if args.pattern:
pattern = "*" + " ".join(args.pattern) + "*"
else:
pattern = "*"
hist = history_manager.search(pattern, raw=raw, output=get_output,
n=limit, unique=args.unique)
print_nums = True
elif args.limit is not _unspecified:
n = 10 if limit is None else limit
hist = history_manager.get_tail(n, raw=raw, output=get_output)
else:
if args.pattern:
range_pattern = "*" + " ".join(args.pattern) + "*"
print_nums = True
hist = history_manager.get_range_by_str(
" ".join(args.range), raw, get_output
)
# We could be displaying the entire history, so let's not try to pull
# it into a list in memory. Anything that needs more space will just
# misalign.
width = 4
for session, lineno, inline in hist:
# Print user history with tabs expanded to 4 spaces. The GUI
# clients use hard tabs for easier usability in auto-indented code,
# but we want to produce PEP-8 compliant history for safe pasting
# into an editor.
if get_output:
inline, output = inline
if range_pattern:
if not fnmatch.fnmatch(inline, range_pattern):
continue
inline = inline.expandtabs(4).rstrip()
multiline = "\n" in inline
line_sep = '\n' if multiline else ' '
if print_nums:
print(u'%s:%s' % (_format_lineno(session, lineno).rjust(width),
line_sep), file=outfile, end=u'')
if pyprompts:
print(u">>> ", end=u"", file=outfile)
if multiline:
inline = "\n... ".join(inline.splitlines()) + "\n..."
print(inline, file=outfile)
if get_output and output:
print(output, file=outfile)
if close_at_end:
outfile.close()
@line_magic
def recall(self, arg):
r"""Repeat a command, or get command to input line for editing.
%recall and %rep are equivalent.
- %recall (no arguments):
Place a string version of last computation result (stored in the
special '_' variable) to the next input prompt. Allows you to create
elaborate command lines without using copy-paste::
In[1]: l = ["hei", "vaan"]
In[2]: "".join(l)
Out[2]: heivaan
In[3]: %recall
In[4]: heivaan_ <== cursor blinking
%recall 45
Place history line 45 on the next input prompt. Use %hist to find
out the number.
%recall 1-4
Combine the specified lines into one cell, and place it on the next
input prompt. See %history for the slice syntax.
%recall foo+bar
If foo+bar can be evaluated in the user namespace, the result is
placed at the next input prompt. Otherwise, the history is searched
for lines which contain that substring, and the most recent one is
placed at the next input prompt.
"""
if not arg: # Last output
self.shell.set_next_input(str(self.shell.user_ns["_"]))
return
# Get history range
histlines = self.shell.history_manager.get_range_by_str(arg)
cmd = "\n".join(x[2] for x in histlines)
if cmd:
self.shell.set_next_input(cmd.rstrip())
return
try: # Variable in user namespace
cmd = str(eval(arg, self.shell.user_ns))
except Exception: # Search for term in history
histlines = self.shell.history_manager.search("*"+arg+"*")
for h in reversed([x[2] for x in histlines]):
if 'recall' in h or 'rep' in h:
continue
self.shell.set_next_input(h.rstrip())
return
else:
self.shell.set_next_input(cmd.rstrip())
return
print("Couldn't evaluate or find in history:", arg)
@line_magic
def rerun(self, parameter_s=''):
"""Re-run previous input
By default, you can specify ranges of input history to be repeated
(as with %history). With no arguments, it will repeat the last line.
Options:
-l <n> : Repeat the last n lines of input, not including the
current command.
-g foo : Repeat the most recent line which contains foo
"""
opts, args = self.parse_options(parameter_s, 'l:g:', mode='string')
if "l" in opts: # Last n lines
try:
n = int(opts["l"])
except ValueError:
print("Number of lines must be an integer")
return
if n == 0:
print("Requested 0 last lines - nothing to run")
return
elif n < 0:
print("Number of lines to rerun cannot be negative")
return
hist = self.shell.history_manager.get_tail(n)
elif "g" in opts: # Search
p = "*"+opts['g']+"*"
hist = list(self.shell.history_manager.search(p))
for l in reversed(hist):
if "rerun" not in l[2]:
hist = [l] # The last match which isn't a %rerun
break
else:
hist = [] # No matches except %rerun
elif args: # Specify history ranges
hist = self.shell.history_manager.get_range_by_str(args)
else: # Last line
hist = self.shell.history_manager.get_tail(1)
hist = [x[2] for x in hist]
if not hist:
print("No lines in history match specification")
return
histlines = "\n".join(hist)
print("=== Executing: ===")
print(histlines)
print("=== Output: ===")
self.shell.run_cell("\n".join(hist), store_history=False)

@ -0,0 +1,195 @@
"""Implementation of magic functions for IPython's own logging.
"""
#-----------------------------------------------------------------------------
# Copyright (c) 2012 The IPython Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
# Stdlib
import os
import sys
# Our own packages
from IPython.core.magic import Magics, magics_class, line_magic
from warnings import warn
from traitlets import Bool
#-----------------------------------------------------------------------------
# Magic implementation classes
#-----------------------------------------------------------------------------
@magics_class
class LoggingMagics(Magics):
"""Magics related to all logging machinery."""
quiet = Bool(False, help=
"""
Suppress output of log state when logging is enabled
"""
).tag(config=True)
@line_magic
def logstart(self, parameter_s=''):
"""Start logging anywhere in a session.
%logstart [-o|-r|-t|-q] [log_name [log_mode]]
If no name is given, it defaults to a file named 'ipython_log.py' in your
current directory, in 'rotate' mode (see below).
'%logstart name' saves to file 'name' in 'backup' mode. It saves your
history up to that point and then continues logging.
%logstart takes a second optional parameter: logging mode. This can be one
of (note that the modes are given unquoted):
append
Keep logging at the end of any existing file.
backup
Rename any existing file to name~ and start name.
global
Append to a single logfile in your home directory.
over
Overwrite any existing log.
rotate
Create rotating logs: name.1~, name.2~, etc.
Options:
-o
log also IPython's output. In this mode, all commands which
generate an Out[NN] prompt are recorded to the logfile, right after
their corresponding input line. The output lines are always
prepended with a '#[Out]# ' marker, so that the log remains valid
Python code.
Since this marker is always the same, filtering only the output from
a log is very easy, using for example a simple awk call::
awk -F'#\\[Out\\]# ' '{if($2) {print $2}}' ipython_log.py
-r
log 'raw' input. Normally, IPython's logs contain the processed
input, so that user lines are logged in their final form, converted
into valid Python. For example, %Exit is logged as
_ip.magic("Exit"). If the -r flag is given, all input is logged
exactly as typed, with no transformations applied.
-t
put timestamps before each input line logged (these are put in
comments).
-q
suppress output of logstate message when logging is invoked
"""
opts,par = self.parse_options(parameter_s,'ortq')
log_output = 'o' in opts
log_raw_input = 'r' in opts
timestamp = 't' in opts
quiet = 'q' in opts
logger = self.shell.logger
# if no args are given, the defaults set in the logger constructor by
# ipython remain valid
if par:
try:
logfname,logmode = par.split()
except:
logfname = par
logmode = 'backup'
else:
logfname = logger.logfname
logmode = logger.logmode
# put logfname into rc struct as if it had been called on the command
# line, so it ends up saved in the log header Save it in case we need
# to restore it...
old_logfile = self.shell.logfile
if logfname:
logfname = os.path.expanduser(logfname)
self.shell.logfile = logfname
loghead = u'# IPython log file\n\n'
try:
logger.logstart(logfname, loghead, logmode, log_output, timestamp,
log_raw_input)
except:
self.shell.logfile = old_logfile
warn("Couldn't start log: %s" % sys.exc_info()[1])
else:
# log input history up to this point, optionally interleaving
# output if requested
if timestamp:
# disable timestamping for the previous history, since we've
# lost those already (no time machine here).
logger.timestamp = False
if log_raw_input:
input_hist = self.shell.history_manager.input_hist_raw
else:
input_hist = self.shell.history_manager.input_hist_parsed
if log_output:
log_write = logger.log_write
output_hist = self.shell.history_manager.output_hist
for n in range(1,len(input_hist)-1):
log_write(input_hist[n].rstrip() + u'\n')
if n in output_hist:
log_write(repr(output_hist[n]),'output')
else:
logger.log_write(u'\n'.join(input_hist[1:]))
logger.log_write(u'\n')
if timestamp:
# re-enable timestamping
logger.timestamp = True
if not (self.quiet or quiet):
print ('Activating auto-logging. '
'Current session state plus future input saved.')
logger.logstate()
@line_magic
def logstop(self, parameter_s=''):
"""Fully stop logging and close log file.
In order to start logging again, a new %logstart call needs to be made,
possibly (though not necessarily) with a new filename, mode and other
options."""
self.shell.logger.logstop()
@line_magic
def logoff(self, parameter_s=''):
"""Temporarily stop logging.
You must have previously started logging."""
self.shell.logger.switch_log(0)
@line_magic
def logon(self, parameter_s=''):
"""Restart logging.
This function is for restarting logging which you've temporarily
stopped with %logoff. For starting logging for the first time, you
must use the %logstart function, which allows you to specify an
optional log filename."""
self.shell.logger.switch_log(1)
@line_magic
def logstate(self, parameter_s=''):
"""Print the status of the logging system."""
self.shell.logger.logstate()

@ -0,0 +1,711 @@
"""Implementation of namespace-related magic functions.
"""
#-----------------------------------------------------------------------------
# Copyright (c) 2012 The IPython Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
# Stdlib
import gc
import re
import sys
# Our own packages
from IPython.core import page
from IPython.core.error import StdinNotImplementedError, UsageError
from IPython.core.magic import Magics, magics_class, line_magic
from IPython.testing.skipdoctest import skip_doctest
from IPython.utils.encoding import DEFAULT_ENCODING
from IPython.utils.openpy import read_py_file
from IPython.utils.path import get_py_filename
#-----------------------------------------------------------------------------
# Magic implementation classes
#-----------------------------------------------------------------------------
@magics_class
class NamespaceMagics(Magics):
"""Magics to manage various aspects of the user's namespace.
These include listing variables, introspecting into them, etc.
"""
@line_magic
def pinfo(self, parameter_s='', namespaces=None):
"""Provide detailed information about an object.
'%pinfo object' is just a synonym for object? or ?object."""
#print 'pinfo par: <%s>' % parameter_s # dbg
# detail_level: 0 -> obj? , 1 -> obj??
detail_level = 0
# We need to detect if we got called as 'pinfo pinfo foo', which can
# happen if the user types 'pinfo foo?' at the cmd line.
pinfo,qmark1,oname,qmark2 = \
re.match(r'(pinfo )?(\?*)(.*?)(\??$)',parameter_s).groups()
if pinfo or qmark1 or qmark2:
detail_level = 1
if "*" in oname:
self.psearch(oname)
else:
self.shell._inspect('pinfo', oname, detail_level=detail_level,
namespaces=namespaces)
@line_magic
def pinfo2(self, parameter_s='', namespaces=None):
"""Provide extra detailed information about an object.
'%pinfo2 object' is just a synonym for object?? or ??object."""
self.shell._inspect('pinfo', parameter_s, detail_level=1,
namespaces=namespaces)
@skip_doctest
@line_magic
def pdef(self, parameter_s='', namespaces=None):
"""Print the call signature for any callable object.
If the object is a class, print the constructor information.
Examples
--------
::
In [3]: %pdef urllib.urlopen
urllib.urlopen(url, data=None, proxies=None)
"""
self.shell._inspect('pdef',parameter_s, namespaces)
@line_magic
def pdoc(self, parameter_s='', namespaces=None):
"""Print the docstring for an object.
If the given object is a class, it will print both the class and the
constructor docstrings."""
self.shell._inspect('pdoc',parameter_s, namespaces)
@line_magic
def psource(self, parameter_s='', namespaces=None):
"""Print (or run through pager) the source code for an object."""
if not parameter_s:
raise UsageError('Missing object name.')
self.shell._inspect('psource',parameter_s, namespaces)
@line_magic
def pfile(self, parameter_s='', namespaces=None):
"""Print (or run through pager) the file where an object is defined.
The file opens at the line where the object definition begins. IPython
will honor the environment variable PAGER if set, and otherwise will
do its best to print the file in a convenient form.
If the given argument is not an object currently defined, IPython will
try to interpret it as a filename (automatically adding a .py extension
if needed). You can thus use %pfile as a syntax highlighting code
viewer."""
# first interpret argument as an object name
out = self.shell._inspect('pfile',parameter_s, namespaces)
# if not, try the input as a filename
if out == 'not found':
try:
filename = get_py_filename(parameter_s)
except IOError as msg:
print(msg)
return
page.page(self.shell.pycolorize(read_py_file(filename, skip_encoding_cookie=False)))
@line_magic
def psearch(self, parameter_s=''):
"""Search for object in namespaces by wildcard.
%psearch [options] PATTERN [OBJECT TYPE]
Note: ? can be used as a synonym for %psearch, at the beginning or at
the end: both a*? and ?a* are equivalent to '%psearch a*'. Still, the
rest of the command line must be unchanged (options come first), so
for example the following forms are equivalent
%psearch -i a* function
-i a* function?
?-i a* function
Arguments:
PATTERN
where PATTERN is a string containing * as a wildcard similar to its
use in a shell. The pattern is matched in all namespaces on the
search path. By default objects starting with a single _ are not
matched, many IPython generated objects have a single
underscore. The default is case insensitive matching. Matching is
also done on the attributes of objects and not only on the objects
in a module.
[OBJECT TYPE]
Is the name of a python type from the types module. The name is
given in lowercase without the ending type, ex. StringType is
written string. By adding a type here only objects matching the
given type are matched. Using all here makes the pattern match all
types (this is the default).
Options:
-a: makes the pattern match even objects whose names start with a
single underscore. These names are normally omitted from the
search.
-i/-c: make the pattern case insensitive/sensitive. If neither of
these options are given, the default is read from your configuration
file, with the option ``InteractiveShell.wildcards_case_sensitive``.
If this option is not specified in your configuration file, IPython's
internal default is to do a case sensitive search.
-e/-s NAMESPACE: exclude/search a given namespace. The pattern you
specify can be searched in any of the following namespaces:
'builtin', 'user', 'user_global','internal', 'alias', where
'builtin' and 'user' are the search defaults. Note that you should
not use quotes when specifying namespaces.
-l: List all available object types for object matching. This function
can be used without arguments.
'Builtin' contains the python module builtin, 'user' contains all
user data, 'alias' only contain the shell aliases and no python
objects, 'internal' contains objects used by IPython. The
'user_global' namespace is only used by embedded IPython instances,
and it contains module-level globals. You can add namespaces to the
search with -s or exclude them with -e (these options can be given
more than once).
Examples
--------
::
%psearch a* -> objects beginning with an a
%psearch -e builtin a* -> objects NOT in the builtin space starting in a
%psearch a* function -> all functions beginning with an a
%psearch re.e* -> objects beginning with an e in module re
%psearch r*.e* -> objects that start with e in modules starting in r
%psearch r*.* string -> all strings in modules beginning with r
Case sensitive search::
%psearch -c a* list all object beginning with lower case a
Show objects beginning with a single _::
%psearch -a _* list objects beginning with a single underscore
List available objects::
%psearch -l list all available object types
"""
# default namespaces to be searched
def_search = ['user_local', 'user_global', 'builtin']
# Process options/args
opts,args = self.parse_options(parameter_s,'cias:e:l',list_all=True)
opt = opts.get
shell = self.shell
psearch = shell.inspector.psearch
# select list object types
list_types = False
if 'l' in opts:
list_types = True
# select case options
if 'i' in opts:
ignore_case = True
elif 'c' in opts:
ignore_case = False
else:
ignore_case = not shell.wildcards_case_sensitive
# Build list of namespaces to search from user options
def_search.extend(opt('s',[]))
ns_exclude = ns_exclude=opt('e',[])
ns_search = [nm for nm in def_search if nm not in ns_exclude]
# Call the actual search
try:
psearch(args,shell.ns_table,ns_search,
show_all=opt('a'),ignore_case=ignore_case, list_types=list_types)
except:
shell.showtraceback()
@skip_doctest
@line_magic
def who_ls(self, parameter_s=''):
"""Return a sorted list of all interactive variables.
If arguments are given, only variables of types matching these
arguments are returned.
Examples
--------
Define two variables and list them with who_ls::
In [1]: alpha = 123
In [2]: beta = 'test'
In [3]: %who_ls
Out[3]: ['alpha', 'beta']
In [4]: %who_ls int
Out[4]: ['alpha']
In [5]: %who_ls str
Out[5]: ['beta']
"""
user_ns = self.shell.user_ns
user_ns_hidden = self.shell.user_ns_hidden
nonmatching = object() # This can never be in user_ns
out = [ i for i in user_ns
if not i.startswith('_') \
and (user_ns[i] is not user_ns_hidden.get(i, nonmatching)) ]
typelist = parameter_s.split()
if typelist:
typeset = set(typelist)
out = [i for i in out if type(user_ns[i]).__name__ in typeset]
out.sort()
return out
@skip_doctest
@line_magic
def who(self, parameter_s=''):
"""Print all interactive variables, with some minimal formatting.
If any arguments are given, only variables whose type matches one of
these are printed. For example::
%who function str
will only list functions and strings, excluding all other types of
variables. To find the proper type names, simply use type(var) at a
command line to see how python prints type names. For example:
::
In [1]: type('hello')\\
Out[1]: <type 'str'>
indicates that the type name for strings is 'str'.
``%who`` always excludes executed names loaded through your configuration
file and things which are internal to IPython.
This is deliberate, as typically you may load many modules and the
purpose of %who is to show you only what you've manually defined.
Examples
--------
Define two variables and list them with who::
In [1]: alpha = 123
In [2]: beta = 'test'
In [3]: %who
alpha beta
In [4]: %who int
alpha
In [5]: %who str
beta
"""
varlist = self.who_ls(parameter_s)
if not varlist:
if parameter_s:
print('No variables match your requested type.')
else:
print('Interactive namespace is empty.')
return
# if we have variables, move on...
count = 0
for i in varlist:
print(i+'\t', end=' ')
count += 1
if count > 8:
count = 0
print()
print()
@skip_doctest
@line_magic
def whos(self, parameter_s=''):
"""Like %who, but gives some extra information about each variable.
The same type filtering of %who can be applied here.
For all variables, the type is printed. Additionally it prints:
- For {},[],(): their length.
- For numpy arrays, a summary with shape, number of
elements, typecode and size in memory.
- Everything else: a string representation, snipping their middle if
too long.
Examples
--------
Define two variables and list them with whos::
In [1]: alpha = 123
In [2]: beta = 'test'
In [3]: %whos
Variable Type Data/Info
--------------------------------
alpha int 123
beta str test
"""
varnames = self.who_ls(parameter_s)
if not varnames:
if parameter_s:
print('No variables match your requested type.')
else:
print('Interactive namespace is empty.')
return
# if we have variables, move on...
# for these types, show len() instead of data:
seq_types = ['dict', 'list', 'tuple']
# for numpy arrays, display summary info
ndarray_type = None
if 'numpy' in sys.modules:
try:
from numpy import ndarray
except ImportError:
pass
else:
ndarray_type = ndarray.__name__
# Find all variable names and types so we can figure out column sizes
# some types are well known and can be shorter
abbrevs = {'IPython.core.macro.Macro' : 'Macro'}
def type_name(v):
tn = type(v).__name__
return abbrevs.get(tn,tn)
varlist = [self.shell.user_ns[n] for n in varnames]
typelist = []
for vv in varlist:
tt = type_name(vv)
if tt=='instance':
typelist.append( abbrevs.get(str(vv.__class__),
str(vv.__class__)))
else:
typelist.append(tt)
# column labels and # of spaces as separator
varlabel = 'Variable'
typelabel = 'Type'
datalabel = 'Data/Info'
colsep = 3
# variable format strings
vformat = "{0:<{varwidth}}{1:<{typewidth}}"
aformat = "%s: %s elems, type `%s`, %s bytes"
# find the size of the columns to format the output nicely
varwidth = max(max(map(len,varnames)), len(varlabel)) + colsep
typewidth = max(max(map(len,typelist)), len(typelabel)) + colsep
# table header
print(varlabel.ljust(varwidth) + typelabel.ljust(typewidth) + \
' '+datalabel+'\n' + '-'*(varwidth+typewidth+len(datalabel)+1))
# and the table itself
kb = 1024
Mb = 1048576 # kb**2
for vname,var,vtype in zip(varnames,varlist,typelist):
print(vformat.format(vname, vtype, varwidth=varwidth, typewidth=typewidth), end=' ')
if vtype in seq_types:
print("n="+str(len(var)))
elif vtype == ndarray_type:
vshape = str(var.shape).replace(',','').replace(' ','x')[1:-1]
if vtype==ndarray_type:
# numpy
vsize = var.size
vbytes = vsize*var.itemsize
vdtype = var.dtype
if vbytes < 100000:
print(aformat % (vshape, vsize, vdtype, vbytes))
else:
print(aformat % (vshape, vsize, vdtype, vbytes), end=' ')
if vbytes < Mb:
print('(%s kb)' % (vbytes/kb,))
else:
print('(%s Mb)' % (vbytes/Mb,))
else:
try:
vstr = str(var)
except UnicodeEncodeError:
vstr = var.encode(DEFAULT_ENCODING,
'backslashreplace')
except:
vstr = "<object with id %d (str() failed)>" % id(var)
vstr = vstr.replace('\n', '\\n')
if len(vstr) < 50:
print(vstr)
else:
print(vstr[:25] + "<...>" + vstr[-25:])
@line_magic
def reset(self, parameter_s=''):
"""Resets the namespace by removing all names defined by the user, if
called without arguments, or by removing some types of objects, such
as everything currently in IPython's In[] and Out[] containers (see
the parameters for details).
Parameters
----------
-f
force reset without asking for confirmation.
-s
'Soft' reset: Only clears your namespace, leaving history intact.
References to objects may be kept. By default (without this option),
we do a 'hard' reset, giving you a new session and removing all
references to objects from the current session.
--aggressive
Try to aggressively remove modules from sys.modules ; this
may allow you to reimport Python modules that have been updated and
pick up changes, but can have unattended consequences.
in
reset input history
out
reset output history
dhist
reset directory history
array
reset only variables that are NumPy arrays
See Also
--------
reset_selective : invoked as ``%reset_selective``
Examples
--------
::
In [6]: a = 1
In [7]: a
Out[7]: 1
In [8]: 'a' in get_ipython().user_ns
Out[8]: True
In [9]: %reset -f
In [1]: 'a' in get_ipython().user_ns
Out[1]: False
In [2]: %reset -f in
Flushing input history
In [3]: %reset -f dhist in
Flushing directory history
Flushing input history
Notes
-----
Calling this magic from clients that do not implement standard input,
such as the ipython notebook interface, will reset the namespace
without confirmation.
"""
opts, args = self.parse_options(parameter_s, "sf", "aggressive", mode="list")
if "f" in opts:
ans = True
else:
try:
ans = self.shell.ask_yes_no(
"Once deleted, variables cannot be recovered. Proceed (y/[n])?",
default='n')
except StdinNotImplementedError:
ans = True
if not ans:
print('Nothing done.')
return
if 's' in opts: # Soft reset
user_ns = self.shell.user_ns
for i in self.who_ls():
del(user_ns[i])
elif len(args) == 0: # Hard reset
self.shell.reset(new_session=False, aggressive=("aggressive" in opts))
# reset in/out/dhist/array: previously extensinions/clearcmd.py
ip = self.shell
user_ns = self.shell.user_ns # local lookup, heavily used
for target in args:
target = target.lower() # make matches case insensitive
if target == 'out':
print("Flushing output cache (%d entries)" % len(user_ns['_oh']))
self.shell.displayhook.flush()
elif target == 'in':
print("Flushing input history")
pc = self.shell.displayhook.prompt_count + 1
for n in range(1, pc):
key = '_i'+repr(n)
user_ns.pop(key,None)
user_ns.update(dict(_i=u'',_ii=u'',_iii=u''))
hm = ip.history_manager
# don't delete these, as %save and %macro depending on the
# length of these lists to be preserved
hm.input_hist_parsed[:] = [''] * pc
hm.input_hist_raw[:] = [''] * pc
# hm has internal machinery for _i,_ii,_iii, clear it out
hm._i = hm._ii = hm._iii = hm._i00 = u''
elif target == 'array':
# Support cleaning up numpy arrays
try:
from numpy import ndarray
# This must be done with items and not iteritems because
# we're going to modify the dict in-place.
for x,val in list(user_ns.items()):
if isinstance(val,ndarray):
del user_ns[x]
except ImportError:
print("reset array only works if Numpy is available.")
elif target == 'dhist':
print("Flushing directory history")
del user_ns['_dh'][:]
else:
print("Don't know how to reset ", end=' ')
print(target + ", please run `%reset?` for details")
gc.collect()
@line_magic
def reset_selective(self, parameter_s=''):
"""Resets the namespace by removing names defined by the user.
Input/Output history are left around in case you need them.
%reset_selective [-f] regex
No action is taken if regex is not included
Options
-f : force reset without asking for confirmation.
See Also
--------
reset : invoked as ``%reset``
Examples
--------
We first fully reset the namespace so your output looks identical to
this example for pedagogical reasons; in practice you do not need a
full reset::
In [1]: %reset -f
Now, with a clean namespace we can make a few variables and use
``%reset_selective`` to only delete names that match our regexp::
In [2]: a=1; b=2; c=3; b1m=4; b2m=5; b3m=6; b4m=7; b2s=8
In [3]: who_ls
Out[3]: ['a', 'b', 'b1m', 'b2m', 'b2s', 'b3m', 'b4m', 'c']
In [4]: %reset_selective -f b[2-3]m
In [5]: who_ls
Out[5]: ['a', 'b', 'b1m', 'b2s', 'b4m', 'c']
In [6]: %reset_selective -f d
In [7]: who_ls
Out[7]: ['a', 'b', 'b1m', 'b2s', 'b4m', 'c']
In [8]: %reset_selective -f c
In [9]: who_ls
Out[9]: ['a', 'b', 'b1m', 'b2s', 'b4m']
In [10]: %reset_selective -f b
In [11]: who_ls
Out[11]: ['a']
Notes
-----
Calling this magic from clients that do not implement standard input,
such as the ipython notebook interface, will reset the namespace
without confirmation.
"""
opts, regex = self.parse_options(parameter_s,'f')
if 'f' in opts:
ans = True
else:
try:
ans = self.shell.ask_yes_no(
"Once deleted, variables cannot be recovered. Proceed (y/[n])? ",
default='n')
except StdinNotImplementedError:
ans = True
if not ans:
print('Nothing done.')
return
user_ns = self.shell.user_ns
if not regex:
print('No regex pattern specified. Nothing done.')
return
else:
try:
m = re.compile(regex)
except TypeError as e:
raise TypeError('regex must be a string or compiled pattern') from e
for i in self.who_ls():
if m.search(i):
del(user_ns[i])
@line_magic
def xdel(self, parameter_s=''):
"""Delete a variable, trying to clear it from anywhere that
IPython's machinery has references to it. By default, this uses
the identity of the named object in the user namespace to remove
references held under other names. The object is also removed
from the output history.
Options
-n : Delete the specified name from all namespaces, without
checking their identity.
"""
opts, varname = self.parse_options(parameter_s,'n')
try:
self.shell.del_var(varname, ('n' in opts))
except (NameError, ValueError) as e:
print(type(e).__name__ +": "+ str(e))

@ -0,0 +1,854 @@
"""Implementation of magic functions for interaction with the OS.
Note: this module is named 'osm' instead of 'os' to avoid a collision with the
builtin.
"""
# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.
import io
import os
import re
import sys
from pprint import pformat
from IPython.core import magic_arguments
from IPython.core import oinspect
from IPython.core import page
from IPython.core.alias import AliasError, Alias
from IPython.core.error import UsageError
from IPython.core.magic import (
Magics, compress_dhist, magics_class, line_magic, cell_magic, line_cell_magic
)
from IPython.testing.skipdoctest import skip_doctest
from IPython.utils.openpy import source_to_unicode
from IPython.utils.process import abbrev_cwd
from IPython.utils.terminal import set_term_title
from traitlets import Bool
from warnings import warn
@magics_class
class OSMagics(Magics):
"""Magics to interact with the underlying OS (shell-type functionality).
"""
cd_force_quiet = Bool(False,
help="Force %cd magic to be quiet even if -q is not passed."
).tag(config=True)
def __init__(self, shell=None, **kwargs):
# Now define isexec in a cross platform manner.
self.is_posix = False
self.execre = None
if os.name == 'posix':
self.is_posix = True
else:
try:
winext = os.environ['pathext'].replace(';','|').replace('.','')
except KeyError:
winext = 'exe|com|bat|py'
try:
self.execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE)
except re.error:
warn("Seems like your pathext environmental "
"variable is malformed. Please check it to "
"enable a proper handle of file extensions "
"managed for your system")
winext = 'exe|com|bat|py'
self.execre = re.compile(r'(.*)\.(%s)$' % winext,re.IGNORECASE)
# call up the chain
super().__init__(shell=shell, **kwargs)
def _isexec_POSIX(self, file):
"""
Test for executable on a POSIX system
"""
if os.access(file.path, os.X_OK):
# will fail on maxOS if access is not X_OK
return file.is_file()
return False
def _isexec_WIN(self, file):
"""
Test for executable file on non POSIX system
"""
return file.is_file() and self.execre.match(file.name) is not None
def isexec(self, file):
"""
Test for executable file on non POSIX system
"""
if self.is_posix:
return self._isexec_POSIX(file)
else:
return self._isexec_WIN(file)
@skip_doctest
@line_magic
def alias(self, parameter_s=''):
"""Define an alias for a system command.
'%alias alias_name cmd' defines 'alias_name' as an alias for 'cmd'
Then, typing 'alias_name params' will execute the system command 'cmd
params' (from your underlying operating system).
Aliases have lower precedence than magic functions and Python normal
variables, so if 'foo' is both a Python variable and an alias, the
alias can not be executed until 'del foo' removes the Python variable.
You can use the %l specifier in an alias definition to represent the
whole line when the alias is called. For example::
In [2]: alias bracket echo "Input in brackets: <%l>"
In [3]: bracket hello world
Input in brackets: <hello world>
You can also define aliases with parameters using %s specifiers (one
per parameter)::
In [1]: alias parts echo first %s second %s
In [2]: %parts A B
first A second B
In [3]: %parts A
Incorrect number of arguments: 2 expected.
parts is an alias to: 'echo first %s second %s'
Note that %l and %s are mutually exclusive. You can only use one or
the other in your aliases.
Aliases expand Python variables just like system calls using ! or !!
do: all expressions prefixed with '$' get expanded. For details of
the semantic rules, see PEP-215:
https://peps.python.org/pep-0215/. This is the library used by
IPython for variable expansion. If you want to access a true shell
variable, an extra $ is necessary to prevent its expansion by
IPython::
In [6]: alias show echo
In [7]: PATH='A Python string'
In [8]: show $PATH
A Python string
In [9]: show $$PATH
/usr/local/lf9560/bin:/usr/local/intel/compiler70/ia32/bin:...
You can use the alias facility to access all of $PATH. See the %rehashx
function, which automatically creates aliases for the contents of your
$PATH.
If called with no parameters, %alias prints the current alias table
for your system. For posix systems, the default aliases are 'cat',
'cp', 'mv', 'rm', 'rmdir', and 'mkdir', and other platform-specific
aliases are added. For windows-based systems, the default aliases are
'copy', 'ddir', 'echo', 'ls', 'ldir', 'mkdir', 'ren', and 'rmdir'.
You can see the definition of alias by adding a question mark in the
end::
In [1]: cat?
Repr: <alias cat for 'cat'>"""
par = parameter_s.strip()
if not par:
aliases = sorted(self.shell.alias_manager.aliases)
# stored = self.shell.db.get('stored_aliases', {} )
# for k, v in stored:
# atab.append(k, v[0])
print("Total number of aliases:", len(aliases))
sys.stdout.flush()
return aliases
# Now try to define a new one
try:
alias,cmd = par.split(None, 1)
except TypeError:
print(oinspect.getdoc(self.alias))
return
try:
self.shell.alias_manager.define_alias(alias, cmd)
except AliasError as e:
print(e)
# end magic_alias
@line_magic
def unalias(self, parameter_s=''):
"""Remove an alias"""
aname = parameter_s.strip()
try:
self.shell.alias_manager.undefine_alias(aname)
except ValueError as e:
print(e)
return
stored = self.shell.db.get('stored_aliases', {} )
if aname in stored:
print("Removing %stored alias",aname)
del stored[aname]
self.shell.db['stored_aliases'] = stored
@line_magic
def rehashx(self, parameter_s=''):
"""Update the alias table with all executable files in $PATH.
rehashx explicitly checks that every entry in $PATH is a file
with execute access (os.X_OK).
Under Windows, it checks executability as a match against a
'|'-separated string of extensions, stored in the IPython config
variable win_exec_ext. This defaults to 'exe|com|bat'.
This function also resets the root module cache of module completer,
used on slow filesystems.
"""
from IPython.core.alias import InvalidAliasError
# for the benefit of module completer in ipy_completers.py
del self.shell.db['rootmodules_cache']
path = [os.path.abspath(os.path.expanduser(p)) for p in
os.environ.get('PATH','').split(os.pathsep)]
syscmdlist = []
savedir = os.getcwd()
# Now walk the paths looking for executables to alias.
try:
# write the whole loop for posix/Windows so we don't have an if in
# the innermost part
if self.is_posix:
for pdir in path:
try:
os.chdir(pdir)
except OSError:
continue
# for python 3.6+ rewrite to: with os.scandir(pdir) as dirlist:
dirlist = os.scandir(path=pdir)
for ff in dirlist:
if self.isexec(ff):
fname = ff.name
try:
# Removes dots from the name since ipython
# will assume names with dots to be python.
if not self.shell.alias_manager.is_alias(fname):
self.shell.alias_manager.define_alias(
fname.replace('.',''), fname)
except InvalidAliasError:
pass
else:
syscmdlist.append(fname)
else:
no_alias = Alias.blacklist
for pdir in path:
try:
os.chdir(pdir)
except OSError:
continue
# for python 3.6+ rewrite to: with os.scandir(pdir) as dirlist:
dirlist = os.scandir(pdir)
for ff in dirlist:
fname = ff.name
base, ext = os.path.splitext(fname)
if self.isexec(ff) and base.lower() not in no_alias:
if ext.lower() == '.exe':
fname = base
try:
# Removes dots from the name since ipython
# will assume names with dots to be python.
self.shell.alias_manager.define_alias(
base.lower().replace('.',''), fname)
except InvalidAliasError:
pass
syscmdlist.append(fname)
self.shell.db['syscmdlist'] = syscmdlist
finally:
os.chdir(savedir)
@skip_doctest
@line_magic
def pwd(self, parameter_s=''):
"""Return the current working directory path.
Examples
--------
::
In [9]: pwd
Out[9]: '/home/tsuser/sprint/ipython'
"""
try:
return os.getcwd()
except FileNotFoundError as e:
raise UsageError("CWD no longer exists - please use %cd to change directory.") from e
@skip_doctest
@line_magic
def cd(self, parameter_s=''):
"""Change the current working directory.
This command automatically maintains an internal list of directories
you visit during your IPython session, in the variable ``_dh``. The
command :magic:`%dhist` shows this history nicely formatted. You can
also do ``cd -<tab>`` to see directory history conveniently.
Usage:
- ``cd 'dir'``: changes to directory 'dir'.
- ``cd -``: changes to the last visited directory.
- ``cd -<n>``: changes to the n-th directory in the directory history.
- ``cd --foo``: change to directory that matches 'foo' in history
- ``cd -b <bookmark_name>``: jump to a bookmark set by %bookmark
- Hitting a tab key after ``cd -b`` allows you to tab-complete
bookmark names.
.. note::
``cd <bookmark_name>`` is enough if there is no directory
``<bookmark_name>``, but a bookmark with the name exists.
Options:
-q Be quiet. Do not print the working directory after the
cd command is executed. By default IPython's cd
command does print this directory, since the default
prompts do not display path information.
.. note::
Note that ``!cd`` doesn't work for this purpose because the shell
where ``!command`` runs is immediately discarded after executing
'command'.
Examples
--------
::
In [10]: cd parent/child
/home/tsuser/parent/child
"""
try:
oldcwd = os.getcwd()
except FileNotFoundError:
# Happens if the CWD has been deleted.
oldcwd = None
numcd = re.match(r'(-)(\d+)$',parameter_s)
# jump in directory history by number
if numcd:
nn = int(numcd.group(2))
try:
ps = self.shell.user_ns['_dh'][nn]
except IndexError:
print('The requested directory does not exist in history.')
return
else:
opts = {}
elif parameter_s.startswith('--'):
ps = None
fallback = None
pat = parameter_s[2:]
dh = self.shell.user_ns['_dh']
# first search only by basename (last component)
for ent in reversed(dh):
if pat in os.path.basename(ent) and os.path.isdir(ent):
ps = ent
break
if fallback is None and pat in ent and os.path.isdir(ent):
fallback = ent
# if we have no last part match, pick the first full path match
if ps is None:
ps = fallback
if ps is None:
print("No matching entry in directory history")
return
else:
opts = {}
else:
opts, ps = self.parse_options(parameter_s, 'qb', mode='string')
# jump to previous
if ps == '-':
try:
ps = self.shell.user_ns['_dh'][-2]
except IndexError as e:
raise UsageError('%cd -: No previous directory to change to.') from e
# jump to bookmark if needed
else:
if not os.path.isdir(ps) or 'b' in opts:
bkms = self.shell.db.get('bookmarks', {})
if ps in bkms:
target = bkms[ps]
print('(bookmark:%s) -> %s' % (ps, target))
ps = target
else:
if 'b' in opts:
raise UsageError("Bookmark '%s' not found. "
"Use '%%bookmark -l' to see your bookmarks." % ps)
# at this point ps should point to the target dir
if ps:
try:
os.chdir(os.path.expanduser(ps))
if hasattr(self.shell, 'term_title') and self.shell.term_title:
set_term_title(self.shell.term_title_format.format(cwd=abbrev_cwd()))
except OSError:
print(sys.exc_info()[1])
else:
cwd = os.getcwd()
dhist = self.shell.user_ns['_dh']
if oldcwd != cwd:
dhist.append(cwd)
self.shell.db['dhist'] = compress_dhist(dhist)[-100:]
else:
os.chdir(self.shell.home_dir)
if hasattr(self.shell, 'term_title') and self.shell.term_title:
set_term_title(self.shell.term_title_format.format(cwd="~"))
cwd = os.getcwd()
dhist = self.shell.user_ns['_dh']
if oldcwd != cwd:
dhist.append(cwd)
self.shell.db['dhist'] = compress_dhist(dhist)[-100:]
if not 'q' in opts and not self.cd_force_quiet and self.shell.user_ns['_dh']:
print(self.shell.user_ns['_dh'][-1])
@line_magic
def env(self, parameter_s=''):
"""Get, set, or list environment variables.
Usage:\\
:``%env``: lists all environment variables/values
:``%env var``: get value for var
:``%env var val``: set value for var
:``%env var=val``: set value for var
:``%env var=$val``: set value for var, using python expansion if possible
"""
if parameter_s.strip():
split = '=' if '=' in parameter_s else ' '
bits = parameter_s.split(split)
if len(bits) == 1:
key = parameter_s.strip()
if key in os.environ:
return os.environ[key]
else:
err = "Environment does not have key: {0}".format(key)
raise UsageError(err)
if len(bits) > 1:
return self.set_env(parameter_s)
env = dict(os.environ)
# hide likely secrets when printing the whole environment
for key in list(env):
if any(s in key.lower() for s in ('key', 'token', 'secret')):
env[key] = '<hidden>'
return env
@line_magic
def set_env(self, parameter_s):
"""Set environment variables. Assumptions are that either "val" is a
name in the user namespace, or val is something that evaluates to a
string.
Usage:\\
%set_env var val: set value for var
%set_env var=val: set value for var
%set_env var=$val: set value for var, using python expansion if possible
"""
split = '=' if '=' in parameter_s else ' '
bits = parameter_s.split(split, 1)
if not parameter_s.strip() or len(bits)<2:
raise UsageError("usage is 'set_env var=val'")
var = bits[0].strip()
val = bits[1].strip()
if re.match(r'.*\s.*', var):
# an environment variable with whitespace is almost certainly
# not what the user intended. what's more likely is the wrong
# split was chosen, ie for "set_env cmd_args A=B", we chose
# '=' for the split and should have chosen ' '. to get around
# this, users should just assign directly to os.environ or use
# standard magic {var} expansion.
err = "refusing to set env var with whitespace: '{0}'"
err = err.format(val)
raise UsageError(err)
os.environ[var] = val
print('env: {0}={1}'.format(var,val))
@line_magic
def pushd(self, parameter_s=''):
"""Place the current dir on stack and change directory.
Usage:\\
%pushd ['dirname']
"""
dir_s = self.shell.dir_stack
tgt = os.path.expanduser(parameter_s)
cwd = os.getcwd().replace(self.shell.home_dir,'~')
if tgt:
self.cd(parameter_s)
dir_s.insert(0,cwd)
return self.shell.run_line_magic('dirs', '')
@line_magic
def popd(self, parameter_s=''):
"""Change to directory popped off the top of the stack.
"""
if not self.shell.dir_stack:
raise UsageError("%popd on empty stack")
top = self.shell.dir_stack.pop(0)
self.cd(top)
print("popd ->",top)
@line_magic
def dirs(self, parameter_s=''):
"""Return the current directory stack."""
return self.shell.dir_stack
@line_magic
def dhist(self, parameter_s=''):
"""Print your history of visited directories.
%dhist -> print full history\\
%dhist n -> print last n entries only\\
%dhist n1 n2 -> print entries between n1 and n2 (n2 not included)\\
This history is automatically maintained by the %cd command, and
always available as the global list variable _dh. You can use %cd -<n>
to go to directory number <n>.
Note that most of time, you should view directory history by entering
cd -<TAB>.
"""
dh = self.shell.user_ns['_dh']
if parameter_s:
try:
args = map(int,parameter_s.split())
except:
self.arg_err(self.dhist)
return
if len(args) == 1:
ini,fin = max(len(dh)-(args[0]),0),len(dh)
elif len(args) == 2:
ini,fin = args
fin = min(fin, len(dh))
else:
self.arg_err(self.dhist)
return
else:
ini,fin = 0,len(dh)
print('Directory history (kept in _dh)')
for i in range(ini, fin):
print("%d: %s" % (i, dh[i]))
@skip_doctest
@line_magic
def sc(self, parameter_s=''):
"""Shell capture - run shell command and capture output (DEPRECATED use !).
DEPRECATED. Suboptimal, retained for backwards compatibility.
You should use the form 'var = !command' instead. Example:
"%sc -l myfiles = ls ~" should now be written as
"myfiles = !ls ~"
myfiles.s, myfiles.l and myfiles.n still apply as documented
below.
--
%sc [options] varname=command
IPython will run the given command using commands.getoutput(), and
will then update the user's interactive namespace with a variable
called varname, containing the value of the call. Your command can
contain shell wildcards, pipes, etc.
The '=' sign in the syntax is mandatory, and the variable name you
supply must follow Python's standard conventions for valid names.
(A special format without variable name exists for internal use)
Options:
-l: list output. Split the output on newlines into a list before
assigning it to the given variable. By default the output is stored
as a single string.
-v: verbose. Print the contents of the variable.
In most cases you should not need to split as a list, because the
returned value is a special type of string which can automatically
provide its contents either as a list (split on newlines) or as a
space-separated string. These are convenient, respectively, either
for sequential processing or to be passed to a shell command.
For example::
# Capture into variable a
In [1]: sc a=ls *py
# a is a string with embedded newlines
In [2]: a
Out[2]: 'setup.py\\nwin32_manual_post_install.py'
# which can be seen as a list:
In [3]: a.l
Out[3]: ['setup.py', 'win32_manual_post_install.py']
# or as a whitespace-separated string:
In [4]: a.s
Out[4]: 'setup.py win32_manual_post_install.py'
# a.s is useful to pass as a single command line:
In [5]: !wc -l $a.s
146 setup.py
130 win32_manual_post_install.py
276 total
# while the list form is useful to loop over:
In [6]: for f in a.l:
...: !wc -l $f
...:
146 setup.py
130 win32_manual_post_install.py
Similarly, the lists returned by the -l option are also special, in
the sense that you can equally invoke the .s attribute on them to
automatically get a whitespace-separated string from their contents::
In [7]: sc -l b=ls *py
In [8]: b
Out[8]: ['setup.py', 'win32_manual_post_install.py']
In [9]: b.s
Out[9]: 'setup.py win32_manual_post_install.py'
In summary, both the lists and strings used for output capture have
the following special attributes::
.l (or .list) : value as list.
.n (or .nlstr): value as newline-separated string.
.s (or .spstr): value as space-separated string.
"""
opts,args = self.parse_options(parameter_s, 'lv')
# Try to get a variable name and command to run
try:
# the variable name must be obtained from the parse_options
# output, which uses shlex.split to strip options out.
var,_ = args.split('=', 1)
var = var.strip()
# But the command has to be extracted from the original input
# parameter_s, not on what parse_options returns, to avoid the
# quote stripping which shlex.split performs on it.
_,cmd = parameter_s.split('=', 1)
except ValueError:
var,cmd = '',''
# If all looks ok, proceed
split = 'l' in opts
out = self.shell.getoutput(cmd, split=split)
if 'v' in opts:
print('%s ==\n%s' % (var, pformat(out)))
if var:
self.shell.user_ns.update({var:out})
else:
return out
@line_cell_magic
def sx(self, line='', cell=None):
"""Shell execute - run shell command and capture output (!! is short-hand).
%sx command
IPython will run the given command using commands.getoutput(), and
return the result formatted as a list (split on '\\n'). Since the
output is _returned_, it will be stored in ipython's regular output
cache Out[N] and in the '_N' automatic variables.
Notes:
1) If an input line begins with '!!', then %sx is automatically
invoked. That is, while::
!ls
causes ipython to simply issue system('ls'), typing::
!!ls
is a shorthand equivalent to::
%sx ls
2) %sx differs from %sc in that %sx automatically splits into a list,
like '%sc -l'. The reason for this is to make it as easy as possible
to process line-oriented shell output via further python commands.
%sc is meant to provide much finer control, but requires more
typing.
3) Just like %sc -l, this is a list with special attributes:
::
.l (or .list) : value as list.
.n (or .nlstr): value as newline-separated string.
.s (or .spstr): value as whitespace-separated string.
This is very useful when trying to use such lists as arguments to
system commands."""
if cell is None:
# line magic
return self.shell.getoutput(line)
else:
opts,args = self.parse_options(line, '', 'out=')
output = self.shell.getoutput(cell)
out_name = opts.get('out', opts.get('o'))
if out_name:
self.shell.user_ns[out_name] = output
else:
return output
system = line_cell_magic('system')(sx)
bang = cell_magic('!')(sx)
@line_magic
def bookmark(self, parameter_s=''):
"""Manage IPython's bookmark system.
%bookmark <name> - set bookmark to current dir
%bookmark <name> <dir> - set bookmark to <dir>
%bookmark -l - list all bookmarks
%bookmark -d <name> - remove bookmark
%bookmark -r - remove all bookmarks
You can later on access a bookmarked folder with::
%cd -b <name>
or simply '%cd <name>' if there is no directory called <name> AND
there is such a bookmark defined.
Your bookmarks persist through IPython sessions, but they are
associated with each profile."""
opts,args = self.parse_options(parameter_s,'drl',mode='list')
if len(args) > 2:
raise UsageError("%bookmark: too many arguments")
bkms = self.shell.db.get('bookmarks',{})
if 'd' in opts:
try:
todel = args[0]
except IndexError as e:
raise UsageError(
"%bookmark -d: must provide a bookmark to delete") from e
else:
try:
del bkms[todel]
except KeyError as e:
raise UsageError(
"%%bookmark -d: Can't delete bookmark '%s'" % todel) from e
elif 'r' in opts:
bkms = {}
elif 'l' in opts:
bks = sorted(bkms)
if bks:
size = max(map(len, bks))
else:
size = 0
fmt = '%-'+str(size)+'s -> %s'
print('Current bookmarks:')
for bk in bks:
print(fmt % (bk, bkms[bk]))
else:
if not args:
raise UsageError("%bookmark: You must specify the bookmark name")
elif len(args)==1:
bkms[args[0]] = os.getcwd()
elif len(args)==2:
bkms[args[0]] = args[1]
self.shell.db['bookmarks'] = bkms
@line_magic
def pycat(self, parameter_s=''):
"""Show a syntax-highlighted file through a pager.
This magic is similar to the cat utility, but it will assume the file
to be Python source and will show it with syntax highlighting.
This magic command can either take a local filename, an url,
an history range (see %history) or a macro as argument.
If no parameter is given, prints out history of current session up to
this point. ::
%pycat myscript.py
%pycat 7-27
%pycat myMacro
%pycat http://www.example.com/myscript.py
"""
try:
cont = self.shell.find_user_code(parameter_s, skip_encoding_cookie=False)
except (ValueError, IOError):
print("Error: no such file, variable, URL, history range or macro")
return
page.page(self.shell.pycolorize(source_to_unicode(cont)))
@magic_arguments.magic_arguments()
@magic_arguments.argument(
'-a', '--append', action='store_true', default=False,
help='Append contents of the cell to an existing file. '
'The file will be created if it does not exist.'
)
@magic_arguments.argument(
'filename', type=str,
help='file to write'
)
@cell_magic
def writefile(self, line, cell):
"""Write the contents of the cell to a file.
The file will be overwritten unless the -a (--append) flag is specified.
"""
args = magic_arguments.parse_argstring(self.writefile, line)
if re.match(r'^(\'.*\')|(".*")$', args.filename):
filename = os.path.expanduser(args.filename[1:-1])
else:
filename = os.path.expanduser(args.filename)
if os.path.exists(filename):
if args.append:
print("Appending to %s" % filename)
else:
print("Overwriting %s" % filename)
else:
print("Writing %s" % filename)
mode = 'a' if args.append else 'w'
with io.open(filename, mode, encoding='utf-8') as f:
f.write(cell)

@ -0,0 +1,112 @@
"""Implementation of packaging-related magic functions.
"""
#-----------------------------------------------------------------------------
# Copyright (c) 2018 The IPython Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
import re
import shlex
import sys
from pathlib import Path
from IPython.core.magic import Magics, magics_class, line_magic
def _is_conda_environment():
"""Return True if the current Python executable is in a conda env"""
# TODO: does this need to change on windows?
return Path(sys.prefix, "conda-meta", "history").exists()
def _get_conda_executable():
"""Find the path to the conda executable"""
# Check if there is a conda executable in the same directory as the Python executable.
# This is the case within conda's root environment.
conda = Path(sys.executable).parent / "conda"
if conda.is_file():
return str(conda)
# Otherwise, attempt to extract the executable from conda history.
# This applies in any conda environment.
history = Path(sys.prefix, "conda-meta", "history").read_text(encoding="utf-8")
match = re.search(
r"^#\s*cmd:\s*(?P<command>.*conda)\s[create|install]",
history,
flags=re.MULTILINE,
)
if match:
return match.groupdict()["command"]
# Fallback: assume conda is available on the system path.
return "conda"
CONDA_COMMANDS_REQUIRING_PREFIX = {
'install', 'list', 'remove', 'uninstall', 'update', 'upgrade',
}
CONDA_COMMANDS_REQUIRING_YES = {
'install', 'remove', 'uninstall', 'update', 'upgrade',
}
CONDA_ENV_FLAGS = {'-p', '--prefix', '-n', '--name'}
CONDA_YES_FLAGS = {'-y', '--y'}
@magics_class
class PackagingMagics(Magics):
"""Magics related to packaging & installation"""
@line_magic
def pip(self, line):
"""Run the pip package manager within the current kernel.
Usage:
%pip install [pkgs]
"""
python = sys.executable
if sys.platform == "win32":
python = '"' + python + '"'
else:
python = shlex.quote(python)
self.shell.system(" ".join([python, "-m", "pip", line]))
print("Note: you may need to restart the kernel to use updated packages.")
@line_magic
def conda(self, line):
"""Run the conda package manager within the current kernel.
Usage:
%conda install [pkgs]
"""
if not _is_conda_environment():
raise ValueError("The python kernel does not appear to be a conda environment. "
"Please use ``%pip install`` instead.")
conda = _get_conda_executable()
args = shlex.split(line)
command = args[0] if len(args) > 0 else ""
args = args[1:] if len(args) > 1 else [""]
extra_args = []
# When the subprocess does not allow us to respond "yes" during the installation,
# we need to insert --yes in the argument list for some commands
stdin_disabled = getattr(self.shell, 'kernel', None) is not None
needs_yes = command in CONDA_COMMANDS_REQUIRING_YES
has_yes = set(args).intersection(CONDA_YES_FLAGS)
if stdin_disabled and needs_yes and not has_yes:
extra_args.append("--yes")
# Add --prefix to point conda installation to the current environment
needs_prefix = command in CONDA_COMMANDS_REQUIRING_PREFIX
has_prefix = set(args).intersection(CONDA_ENV_FLAGS)
if needs_prefix and not has_prefix:
extra_args.extend(["--prefix", sys.prefix])
self.shell.system(' '.join([conda, command] + extra_args + args))
print("\nNote: you may need to restart the kernel to use updated packages.")

@ -0,0 +1,169 @@
"""Implementation of magic functions for matplotlib/pylab support.
"""
#-----------------------------------------------------------------------------
# Copyright (c) 2012 The IPython Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
# Our own packages
from traitlets.config.application import Application
from IPython.core import magic_arguments
from IPython.core.magic import Magics, magics_class, line_magic
from IPython.testing.skipdoctest import skip_doctest
from warnings import warn
from IPython.core.pylabtools import backends
#-----------------------------------------------------------------------------
# Magic implementation classes
#-----------------------------------------------------------------------------
magic_gui_arg = magic_arguments.argument(
'gui', nargs='?',
help="""Name of the matplotlib backend to use %s.
If given, the corresponding matplotlib backend is used,
otherwise it will be matplotlib's default
(which you can set in your matplotlib config file).
""" % str(tuple(sorted(backends.keys())))
)
@magics_class
class PylabMagics(Magics):
"""Magics related to matplotlib's pylab support"""
@skip_doctest
@line_magic
@magic_arguments.magic_arguments()
@magic_arguments.argument('-l', '--list', action='store_true',
help='Show available matplotlib backends')
@magic_gui_arg
def matplotlib(self, line=''):
"""Set up matplotlib to work interactively.
This function lets you activate matplotlib interactive support
at any point during an IPython session. It does not import anything
into the interactive namespace.
If you are using the inline matplotlib backend in the IPython Notebook
you can set which figure formats are enabled using the following::
In [1]: from IPython.display import set_matplotlib_formats
In [2]: set_matplotlib_formats('pdf', 'svg')
The default for inline figures sets `bbox_inches` to 'tight'. This can
cause discrepancies between the displayed image and the identical
image created using `savefig`. This behavior can be disabled using the
`%config` magic::
In [3]: %config InlineBackend.print_figure_kwargs = {'bbox_inches':None}
In addition, see the docstring of
`IPython.display.set_matplotlib_formats` and
`IPython.display.set_matplotlib_close` for more information on
changing additional behaviors of the inline backend.
Examples
--------
To enable the inline backend for usage with the IPython Notebook::
In [1]: %matplotlib inline
In this case, where the matplotlib default is TkAgg::
In [2]: %matplotlib
Using matplotlib backend: TkAgg
But you can explicitly request a different GUI backend::
In [3]: %matplotlib qt
You can list the available backends using the -l/--list option::
In [4]: %matplotlib --list
Available matplotlib backends: ['osx', 'qt4', 'qt5', 'gtk3', 'gtk4', 'notebook', 'wx', 'qt', 'nbagg',
'gtk', 'tk', 'inline']
"""
args = magic_arguments.parse_argstring(self.matplotlib, line)
if args.list:
backends_list = list(backends.keys())
print("Available matplotlib backends: %s" % backends_list)
else:
gui, backend = self.shell.enable_matplotlib(args.gui.lower() if isinstance(args.gui, str) else args.gui)
self._show_matplotlib_backend(args.gui, backend)
@skip_doctest
@line_magic
@magic_arguments.magic_arguments()
@magic_arguments.argument(
'--no-import-all', action='store_true', default=None,
help="""Prevent IPython from performing ``import *`` into the interactive namespace.
You can govern the default behavior of this flag with the
InteractiveShellApp.pylab_import_all configurable.
"""
)
@magic_gui_arg
def pylab(self, line=''):
"""Load numpy and matplotlib to work interactively.
This function lets you activate pylab (matplotlib, numpy and
interactive support) at any point during an IPython session.
%pylab makes the following imports::
import numpy
import matplotlib
from matplotlib import pylab, mlab, pyplot
np = numpy
plt = pyplot
from IPython.display import display
from IPython.core.pylabtools import figsize, getfigs
from pylab import *
from numpy import *
If you pass `--no-import-all`, the last two `*` imports will be excluded.
See the %matplotlib magic for more details about activating matplotlib
without affecting the interactive namespace.
"""
args = magic_arguments.parse_argstring(self.pylab, line)
if args.no_import_all is None:
# get default from Application
if Application.initialized():
app = Application.instance()
try:
import_all = app.pylab_import_all
except AttributeError:
import_all = True
else:
# nothing specified, no app - default True
import_all = True
else:
# invert no-import flag
import_all = not args.no_import_all
gui, backend, clobbered = self.shell.enable_pylab(args.gui, import_all=import_all)
self._show_matplotlib_backend(args.gui, backend)
print(
"%pylab is deprecated, use %matplotlib inline and import the required libraries."
)
print("Populating the interactive namespace from numpy and matplotlib")
if clobbered:
warn("pylab import has clobbered these variables: %s" % clobbered +
"\n`%matplotlib` prevents importing * from pylab and numpy"
)
def _show_matplotlib_backend(self, gui, backend):
"""show matplotlib message backend message"""
if not gui or gui == 'auto':
print("Using matplotlib backend: %s" % backend)

@ -0,0 +1,362 @@
"""Magic functions for running cells in various scripts."""
# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.
import asyncio
import atexit
import errno
import os
import signal
import sys
import time
from subprocess import CalledProcessError
from threading import Thread
from traitlets import Any, Dict, List, default
from IPython.core import magic_arguments
from IPython.core.async_helpers import _AsyncIOProxy
from IPython.core.magic import Magics, cell_magic, line_magic, magics_class
from IPython.utils.process import arg_split
#-----------------------------------------------------------------------------
# Magic implementation classes
#-----------------------------------------------------------------------------
def script_args(f):
"""single decorator for adding script args"""
args = [
magic_arguments.argument(
'--out', type=str,
help="""The variable in which to store stdout from the script.
If the script is backgrounded, this will be the stdout *pipe*,
instead of the stderr text itself and will not be auto closed.
"""
),
magic_arguments.argument(
'--err', type=str,
help="""The variable in which to store stderr from the script.
If the script is backgrounded, this will be the stderr *pipe*,
instead of the stderr text itself and will not be autoclosed.
"""
),
magic_arguments.argument(
'--bg', action="store_true",
help="""Whether to run the script in the background.
If given, the only way to see the output of the command is
with --out/err.
"""
),
magic_arguments.argument(
'--proc', type=str,
help="""The variable in which to store Popen instance.
This is used only when --bg option is given.
"""
),
magic_arguments.argument(
'--no-raise-error', action="store_false", dest='raise_error',
help="""Whether you should raise an error message in addition to
a stream on stderr if you get a nonzero exit code.
""",
),
]
for arg in args:
f = arg(f)
return f
@magics_class
class ScriptMagics(Magics):
"""Magics for talking to scripts
This defines a base `%%script` cell magic for running a cell
with a program in a subprocess, and registers a few top-level
magics that call %%script with common interpreters.
"""
event_loop = Any(
help="""
The event loop on which to run subprocesses
Not the main event loop,
because we want to be able to make blocking calls
and have certain requirements we don't want to impose on the main loop.
"""
)
script_magics = List(
help="""Extra script cell magics to define
This generates simple wrappers of `%%script foo` as `%%foo`.
If you want to add script magics that aren't on your path,
specify them in script_paths
""",
).tag(config=True)
@default('script_magics')
def _script_magics_default(self):
"""default to a common list of programs"""
defaults = [
'sh',
'bash',
'perl',
'ruby',
'python',
'python2',
'python3',
'pypy',
]
if os.name == 'nt':
defaults.extend([
'cmd',
])
return defaults
script_paths = Dict(
help="""Dict mapping short 'ruby' names to full paths, such as '/opt/secret/bin/ruby'
Only necessary for items in script_magics where the default path will not
find the right interpreter.
"""
).tag(config=True)
def __init__(self, shell=None):
super(ScriptMagics, self).__init__(shell=shell)
self._generate_script_magics()
self.bg_processes = []
atexit.register(self.kill_bg_processes)
def __del__(self):
self.kill_bg_processes()
def _generate_script_magics(self):
cell_magics = self.magics['cell']
for name in self.script_magics:
cell_magics[name] = self._make_script_magic(name)
def _make_script_magic(self, name):
"""make a named magic, that calls %%script with a particular program"""
# expand to explicit path if necessary:
script = self.script_paths.get(name, name)
@magic_arguments.magic_arguments()
@script_args
def named_script_magic(line, cell):
# if line, add it as cl-flags
if line:
line = "%s %s" % (script, line)
else:
line = script
return self.shebang(line, cell)
# write a basic docstring:
named_script_magic.__doc__ = \
"""%%{name} script magic
Run cells with {script} in a subprocess.
This is a shortcut for `%%script {script}`
""".format(**locals())
return named_script_magic
@magic_arguments.magic_arguments()
@script_args
@cell_magic("script")
def shebang(self, line, cell):
"""Run a cell via a shell command
The `%%script` line is like the #! line of script,
specifying a program (bash, perl, ruby, etc.) with which to run.
The rest of the cell is run by that program.
Examples
--------
::
In [1]: %%script bash
...: for i in 1 2 3; do
...: echo $i
...: done
1
2
3
"""
# Create the event loop in which to run script magics
# this operates on a background thread
if self.event_loop is None:
if sys.platform == "win32":
# don't override the current policy,
# just create an event loop
event_loop = asyncio.WindowsProactorEventLoopPolicy().new_event_loop()
else:
event_loop = asyncio.new_event_loop()
self.event_loop = event_loop
# start the loop in a background thread
asyncio_thread = Thread(target=event_loop.run_forever, daemon=True)
asyncio_thread.start()
else:
event_loop = self.event_loop
def in_thread(coro):
"""Call a coroutine on the asyncio thread"""
return asyncio.run_coroutine_threadsafe(coro, event_loop).result()
async def _handle_stream(stream, stream_arg, file_object):
while True:
line = (await stream.readline()).decode("utf8")
if not line:
break
if stream_arg:
self.shell.user_ns[stream_arg] = line
else:
file_object.write(line)
file_object.flush()
async def _stream_communicate(process, cell):
process.stdin.write(cell)
process.stdin.close()
stdout_task = asyncio.create_task(
_handle_stream(process.stdout, args.out, sys.stdout)
)
stderr_task = asyncio.create_task(
_handle_stream(process.stderr, args.err, sys.stderr)
)
await asyncio.wait([stdout_task, stderr_task])
await process.wait()
argv = arg_split(line, posix=not sys.platform.startswith("win"))
args, cmd = self.shebang.parser.parse_known_args(argv)
try:
p = in_thread(
asyncio.create_subprocess_exec(
*cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
stdin=asyncio.subprocess.PIPE,
)
)
except OSError as e:
if e.errno == errno.ENOENT:
print("Couldn't find program: %r" % cmd[0])
return
else:
raise
if not cell.endswith('\n'):
cell += '\n'
cell = cell.encode('utf8', 'replace')
if args.bg:
self.bg_processes.append(p)
self._gc_bg_processes()
to_close = []
if args.out:
self.shell.user_ns[args.out] = _AsyncIOProxy(p.stdout, event_loop)
else:
to_close.append(p.stdout)
if args.err:
self.shell.user_ns[args.err] = _AsyncIOProxy(p.stderr, event_loop)
else:
to_close.append(p.stderr)
event_loop.call_soon_threadsafe(
lambda: asyncio.Task(self._run_script(p, cell, to_close))
)
if args.proc:
proc_proxy = _AsyncIOProxy(p, event_loop)
proc_proxy.stdout = _AsyncIOProxy(p.stdout, event_loop)
proc_proxy.stderr = _AsyncIOProxy(p.stderr, event_loop)
self.shell.user_ns[args.proc] = proc_proxy
return
try:
in_thread(_stream_communicate(p, cell))
except KeyboardInterrupt:
try:
p.send_signal(signal.SIGINT)
in_thread(asyncio.wait_for(p.wait(), timeout=0.1))
if p.returncode is not None:
print("Process is interrupted.")
return
p.terminate()
in_thread(asyncio.wait_for(p.wait(), timeout=0.1))
if p.returncode is not None:
print("Process is terminated.")
return
p.kill()
print("Process is killed.")
except OSError:
pass
except Exception as e:
print("Error while terminating subprocess (pid=%i): %s" % (p.pid, e))
return
if args.raise_error and p.returncode != 0:
# If we get here and p.returncode is still None, we must have
# killed it but not yet seen its return code. We don't wait for it,
# in case it's stuck in uninterruptible sleep. -9 = SIGKILL
rc = p.returncode or -9
raise CalledProcessError(rc, cell)
shebang.__skip_doctest__ = os.name != "posix"
async def _run_script(self, p, cell, to_close):
"""callback for running the script in the background"""
p.stdin.write(cell)
await p.stdin.drain()
p.stdin.close()
await p.stdin.wait_closed()
await p.wait()
# asyncio read pipes have no close
# but we should drain the data anyway
for s in to_close:
await s.read()
self._gc_bg_processes()
@line_magic("killbgscripts")
def killbgscripts(self, _nouse_=''):
"""Kill all BG processes started by %%script and its family."""
self.kill_bg_processes()
print("All background processes were killed.")
def kill_bg_processes(self):
"""Kill all BG processes which are still running."""
if not self.bg_processes:
return
for p in self.bg_processes:
if p.returncode is None:
try:
p.send_signal(signal.SIGINT)
except:
pass
time.sleep(0.1)
self._gc_bg_processes()
if not self.bg_processes:
return
for p in self.bg_processes:
if p.returncode is None:
try:
p.terminate()
except:
pass
time.sleep(0.1)
self._gc_bg_processes()
if not self.bg_processes:
return
for p in self.bg_processes:
if p.returncode is None:
try:
p.kill()
except:
pass
self._gc_bg_processes()
def _gc_bg_processes(self):
self.bg_processes = [p for p in self.bg_processes if p.returncode is None]

@ -0,0 +1,348 @@
# encoding: utf-8
"""
Paging capabilities for IPython.core
Notes
-----
For now this uses IPython hooks, so it can't be in IPython.utils. If we can get
rid of that dependency, we could move it there.
-----
"""
# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.
import os
import io
import re
import sys
import tempfile
import subprocess
from io import UnsupportedOperation
from pathlib import Path
from IPython import get_ipython
from IPython.display import display
from IPython.core.error import TryNext
from IPython.utils.data import chop
from IPython.utils.process import system
from IPython.utils.terminal import get_terminal_size
from IPython.utils import py3compat
def display_page(strng, start=0, screen_lines=25):
"""Just display, no paging. screen_lines is ignored."""
if isinstance(strng, dict):
data = strng
else:
if start:
strng = u'\n'.join(strng.splitlines()[start:])
data = { 'text/plain': strng }
display(data, raw=True)
def as_hook(page_func):
"""Wrap a pager func to strip the `self` arg
so it can be called as a hook.
"""
return lambda self, *args, **kwargs: page_func(*args, **kwargs)
esc_re = re.compile(r"(\x1b[^m]+m)")
def page_dumb(strng, start=0, screen_lines=25):
"""Very dumb 'pager' in Python, for when nothing else works.
Only moves forward, same interface as page(), except for pager_cmd and
mode.
"""
if isinstance(strng, dict):
strng = strng.get('text/plain', '')
out_ln = strng.splitlines()[start:]
screens = chop(out_ln,screen_lines-1)
if len(screens) == 1:
print(os.linesep.join(screens[0]))
else:
last_escape = ""
for scr in screens[0:-1]:
hunk = os.linesep.join(scr)
print(last_escape + hunk)
if not page_more():
return
esc_list = esc_re.findall(hunk)
if len(esc_list) > 0:
last_escape = esc_list[-1]
print(last_escape + os.linesep.join(screens[-1]))
def _detect_screen_size(screen_lines_def):
"""Attempt to work out the number of lines on the screen.
This is called by page(). It can raise an error (e.g. when run in the
test suite), so it's separated out so it can easily be called in a try block.
"""
TERM = os.environ.get('TERM',None)
if not((TERM=='xterm' or TERM=='xterm-color') and sys.platform != 'sunos5'):
# curses causes problems on many terminals other than xterm, and
# some termios calls lock up on Sun OS5.
return screen_lines_def
try:
import termios
import curses
except ImportError:
return screen_lines_def
# There is a bug in curses, where *sometimes* it fails to properly
# initialize, and then after the endwin() call is made, the
# terminal is left in an unusable state. Rather than trying to
# check every time for this (by requesting and comparing termios
# flags each time), we just save the initial terminal state and
# unconditionally reset it every time. It's cheaper than making
# the checks.
try:
term_flags = termios.tcgetattr(sys.stdout)
except termios.error as err:
# can fail on Linux 2.6, pager_page will catch the TypeError
raise TypeError('termios error: {0}'.format(err)) from err
try:
scr = curses.initscr()
except AttributeError:
# Curses on Solaris may not be complete, so we can't use it there
return screen_lines_def
screen_lines_real,screen_cols = scr.getmaxyx()
curses.endwin()
# Restore terminal state in case endwin() didn't.
termios.tcsetattr(sys.stdout,termios.TCSANOW,term_flags)
# Now we have what we needed: the screen size in rows/columns
return screen_lines_real
#print '***Screen size:',screen_lines_real,'lines x',\
#screen_cols,'columns.' # dbg
def pager_page(strng, start=0, screen_lines=0, pager_cmd=None):
"""Display a string, piping through a pager after a certain length.
strng can be a mime-bundle dict, supplying multiple representations,
keyed by mime-type.
The screen_lines parameter specifies the number of *usable* lines of your
terminal screen (total lines minus lines you need to reserve to show other
information).
If you set screen_lines to a number <=0, page() will try to auto-determine
your screen size and will only use up to (screen_size+screen_lines) for
printing, paging after that. That is, if you want auto-detection but need
to reserve the bottom 3 lines of the screen, use screen_lines = -3, and for
auto-detection without any lines reserved simply use screen_lines = 0.
If a string won't fit in the allowed lines, it is sent through the
specified pager command. If none given, look for PAGER in the environment,
and ultimately default to less.
If no system pager works, the string is sent through a 'dumb pager'
written in python, very simplistic.
"""
# for compatibility with mime-bundle form:
if isinstance(strng, dict):
strng = strng['text/plain']
# Ugly kludge, but calling curses.initscr() flat out crashes in emacs
TERM = os.environ.get('TERM','dumb')
if TERM in ['dumb','emacs'] and os.name != 'nt':
print(strng)
return
# chop off the topmost part of the string we don't want to see
str_lines = strng.splitlines()[start:]
str_toprint = os.linesep.join(str_lines)
num_newlines = len(str_lines)
len_str = len(str_toprint)
# Dumb heuristics to guesstimate number of on-screen lines the string
# takes. Very basic, but good enough for docstrings in reasonable
# terminals. If someone later feels like refining it, it's not hard.
numlines = max(num_newlines,int(len_str/80)+1)
screen_lines_def = get_terminal_size()[1]
# auto-determine screen size
if screen_lines <= 0:
try:
screen_lines += _detect_screen_size(screen_lines_def)
except (TypeError, UnsupportedOperation):
print(str_toprint)
return
#print 'numlines',numlines,'screenlines',screen_lines # dbg
if numlines <= screen_lines :
#print '*** normal print' # dbg
print(str_toprint)
else:
# Try to open pager and default to internal one if that fails.
# All failure modes are tagged as 'retval=1', to match the return
# value of a failed system command. If any intermediate attempt
# sets retval to 1, at the end we resort to our own page_dumb() pager.
pager_cmd = get_pager_cmd(pager_cmd)
pager_cmd += ' ' + get_pager_start(pager_cmd,start)
if os.name == 'nt':
if pager_cmd.startswith('type'):
# The default WinXP 'type' command is failing on complex strings.
retval = 1
else:
fd, tmpname = tempfile.mkstemp('.txt')
tmppath = Path(tmpname)
try:
os.close(fd)
with tmppath.open("wt", encoding="utf-8") as tmpfile:
tmpfile.write(strng)
cmd = "%s < %s" % (pager_cmd, tmppath)
# tmpfile needs to be closed for windows
if os.system(cmd):
retval = 1
else:
retval = None
finally:
Path.unlink(tmppath)
else:
try:
retval = None
# Emulate os.popen, but redirect stderr
proc = subprocess.Popen(
pager_cmd,
shell=True,
stdin=subprocess.PIPE,
stderr=subprocess.DEVNULL,
)
pager = os._wrap_close(
io.TextIOWrapper(proc.stdin, encoding="utf-8"), proc
)
try:
pager_encoding = pager.encoding or sys.stdout.encoding
pager.write(strng)
finally:
retval = pager.close()
except IOError as msg: # broken pipe when user quits
if msg.args == (32, 'Broken pipe'):
retval = None
else:
retval = 1
except OSError:
# Other strange problems, sometimes seen in Win2k/cygwin
retval = 1
if retval is not None:
page_dumb(strng,screen_lines=screen_lines)
def page(data, start=0, screen_lines=0, pager_cmd=None):
"""Display content in a pager, piping through a pager after a certain length.
data can be a mime-bundle dict, supplying multiple representations,
keyed by mime-type, or text.
Pager is dispatched via the `show_in_pager` IPython hook.
If no hook is registered, `pager_page` will be used.
"""
# Some routines may auto-compute start offsets incorrectly and pass a
# negative value. Offset to 0 for robustness.
start = max(0, start)
# first, try the hook
ip = get_ipython()
if ip:
try:
ip.hooks.show_in_pager(data, start=start, screen_lines=screen_lines)
return
except TryNext:
pass
# fallback on default pager
return pager_page(data, start, screen_lines, pager_cmd)
def page_file(fname, start=0, pager_cmd=None):
"""Page a file, using an optional pager command and starting line.
"""
pager_cmd = get_pager_cmd(pager_cmd)
pager_cmd += ' ' + get_pager_start(pager_cmd,start)
try:
if os.environ['TERM'] in ['emacs','dumb']:
raise EnvironmentError
system(pager_cmd + ' ' + fname)
except:
try:
if start > 0:
start -= 1
page(open(fname, encoding="utf-8").read(), start)
except:
print('Unable to show file',repr(fname))
def get_pager_cmd(pager_cmd=None):
"""Return a pager command.
Makes some attempts at finding an OS-correct one.
"""
if os.name == 'posix':
default_pager_cmd = 'less -R' # -R for color control sequences
elif os.name in ['nt','dos']:
default_pager_cmd = 'type'
if pager_cmd is None:
try:
pager_cmd = os.environ['PAGER']
except:
pager_cmd = default_pager_cmd
if pager_cmd == 'less' and '-r' not in os.environ.get('LESS', '').lower():
pager_cmd += ' -R'
return pager_cmd
def get_pager_start(pager, start):
"""Return the string for paging files with an offset.
This is the '+N' argument which less and more (under Unix) accept.
"""
if pager in ['less','more']:
if start:
start_string = '+' + str(start)
else:
start_string = ''
else:
start_string = ''
return start_string
# (X)emacs on win32 doesn't like to be bypassed with msvcrt.getch()
if os.name == 'nt' and os.environ.get('TERM','dumb') != 'emacs':
import msvcrt
def page_more():
""" Smart pausing between pages
@return: True if need print more lines, False if quit
"""
sys.stdout.write('---Return to continue, q to quit--- ')
ans = msvcrt.getwch()
if ans in ("q", "Q"):
result = False
else:
result = True
sys.stdout.write("\b"*37 + " "*37 + "\b"*37)
return result
else:
def page_more():
ans = py3compat.input('---Return to continue, q to quit--- ')
if ans.lower().startswith('q'):
return False
else:
return True

@ -0,0 +1,55 @@
# -*- coding: utf-8 -*-
"""Payload system for IPython.
Authors:
* Fernando Perez
* Brian Granger
"""
#-----------------------------------------------------------------------------
# Copyright (C) 2008-2011 The IPython Development Team
#
# Distributed under the terms of the BSD License. The full license is in
# the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------
#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
from traitlets.config.configurable import Configurable
from traitlets import List
#-----------------------------------------------------------------------------
# Main payload class
#-----------------------------------------------------------------------------
class PayloadManager(Configurable):
_payload = List([])
def write_payload(self, data, single=True):
"""Include or update the specified `data` payload in the PayloadManager.
If a previous payload with the same source exists and `single` is True,
it will be overwritten with the new one.
"""
if not isinstance(data, dict):
raise TypeError('Each payload write must be a dict, got: %r' % data)
if single and 'source' in data:
source = data['source']
for i, pl in enumerate(self._payload):
if 'source' in pl and pl['source'] == source:
self._payload[i] = data
return
self._payload.append(data)
def read_payload(self):
return self._payload
def clear_payload(self):
self._payload = []

@ -0,0 +1,51 @@
# encoding: utf-8
"""A payload based version of page."""
# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.
import warnings
from IPython.core.getipython import get_ipython
def page(strng, start=0, screen_lines=0, pager_cmd=None):
"""Print a string, piping through a pager.
This version ignores the screen_lines and pager_cmd arguments and uses
IPython's payload system instead.
Parameters
----------
strng : str or mime-dict
Text to page, or a mime-type keyed dict of already formatted data.
start : int
Starting line at which to place the display.
"""
# Some routines may auto-compute start offsets incorrectly and pass a
# negative value. Offset to 0 for robustness.
start = max(0, start)
shell = get_ipython()
if isinstance(strng, dict):
data = strng
else:
data = {'text/plain' : strng}
payload = dict(
source='page',
data=data,
start=start,
)
shell.payload_manager.write_payload(payload)
def install_payload_page():
"""DEPRECATED, use show_in_pager hook
Install this version of page as IPython.core.page.page.
"""
warnings.warn("""install_payload_page is deprecated.
Use `ip.set_hook('show_in_pager, page.as_hook(payloadpage.page))`
""")
from IPython.core import page as corepage
corepage.page = page

@ -0,0 +1,698 @@
# encoding: utf-8
"""
Prefiltering components.
Prefilters transform user input before it is exec'd by Python. These
transforms are used to implement additional syntax such as !ls and %magic.
"""
# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.
from keyword import iskeyword
import re
from .autocall import IPyAutocall
from traitlets.config.configurable import Configurable
from .inputtransformer2 import (
ESC_MAGIC,
ESC_QUOTE,
ESC_QUOTE2,
ESC_PAREN,
)
from .macro import Macro
from .splitinput import LineInfo
from traitlets import (
List, Integer, Unicode, Bool, Instance, CRegExp
)
#-----------------------------------------------------------------------------
# Global utilities, errors and constants
#-----------------------------------------------------------------------------
class PrefilterError(Exception):
pass
# RegExp to identify potential function names
re_fun_name = re.compile(r'[^\W\d]([\w.]*) *$')
# RegExp to exclude strings with this start from autocalling. In
# particular, all binary operators should be excluded, so that if foo is
# callable, foo OP bar doesn't become foo(OP bar), which is invalid. The
# characters '!=()' don't need to be checked for, as the checkPythonChars
# routine explicitly does so, to catch direct calls and rebindings of
# existing names.
# Warning: the '-' HAS TO BE AT THE END of the first group, otherwise
# it affects the rest of the group in square brackets.
re_exclude_auto = re.compile(r'^[,&^\|\*/\+-]'
r'|^is |^not |^in |^and |^or ')
# try to catch also methods for stuff in lists/tuples/dicts: off
# (experimental). For this to work, the line_split regexp would need
# to be modified so it wouldn't break things at '['. That line is
# nasty enough that I shouldn't change it until I can test it _well_.
#self.re_fun_name = re.compile (r'[a-zA-Z_]([a-zA-Z0-9_.\[\]]*) ?$')
# Handler Check Utilities
def is_shadowed(identifier, ip):
"""Is the given identifier defined in one of the namespaces which shadow
the alias and magic namespaces? Note that an identifier is different
than ifun, because it can not contain a '.' character."""
# This is much safer than calling ofind, which can change state
return (identifier in ip.user_ns \
or identifier in ip.user_global_ns \
or identifier in ip.ns_table['builtin']\
or iskeyword(identifier))
#-----------------------------------------------------------------------------
# Main Prefilter manager
#-----------------------------------------------------------------------------
class PrefilterManager(Configurable):
"""Main prefilter component.
The IPython prefilter is run on all user input before it is run. The
prefilter consumes lines of input and produces transformed lines of
input.
The implementation consists of two phases:
1. Transformers
2. Checkers and handlers
Over time, we plan on deprecating the checkers and handlers and doing
everything in the transformers.
The transformers are instances of :class:`PrefilterTransformer` and have
a single method :meth:`transform` that takes a line and returns a
transformed line. The transformation can be accomplished using any
tool, but our current ones use regular expressions for speed.
After all the transformers have been run, the line is fed to the checkers,
which are instances of :class:`PrefilterChecker`. The line is passed to
the :meth:`check` method, which either returns `None` or a
:class:`PrefilterHandler` instance. If `None` is returned, the other
checkers are tried. If an :class:`PrefilterHandler` instance is returned,
the line is passed to the :meth:`handle` method of the returned
handler and no further checkers are tried.
Both transformers and checkers have a `priority` attribute, that determines
the order in which they are called. Smaller priorities are tried first.
Both transformers and checkers also have `enabled` attribute, which is
a boolean that determines if the instance is used.
Users or developers can change the priority or enabled attribute of
transformers or checkers, but they must call the :meth:`sort_checkers`
or :meth:`sort_transformers` method after changing the priority.
"""
multi_line_specials = Bool(True).tag(config=True)
shell = Instance('IPython.core.interactiveshell.InteractiveShellABC', allow_none=True)
def __init__(self, shell=None, **kwargs):
super(PrefilterManager, self).__init__(shell=shell, **kwargs)
self.shell = shell
self._transformers = []
self.init_handlers()
self.init_checkers()
#-------------------------------------------------------------------------
# API for managing transformers
#-------------------------------------------------------------------------
def sort_transformers(self):
"""Sort the transformers by priority.
This must be called after the priority of a transformer is changed.
The :meth:`register_transformer` method calls this automatically.
"""
self._transformers.sort(key=lambda x: x.priority)
@property
def transformers(self):
"""Return a list of checkers, sorted by priority."""
return self._transformers
def register_transformer(self, transformer):
"""Register a transformer instance."""
if transformer not in self._transformers:
self._transformers.append(transformer)
self.sort_transformers()
def unregister_transformer(self, transformer):
"""Unregister a transformer instance."""
if transformer in self._transformers:
self._transformers.remove(transformer)
#-------------------------------------------------------------------------
# API for managing checkers
#-------------------------------------------------------------------------
def init_checkers(self):
"""Create the default checkers."""
self._checkers = []
for checker in _default_checkers:
checker(
shell=self.shell, prefilter_manager=self, parent=self
)
def sort_checkers(self):
"""Sort the checkers by priority.
This must be called after the priority of a checker is changed.
The :meth:`register_checker` method calls this automatically.
"""
self._checkers.sort(key=lambda x: x.priority)
@property
def checkers(self):
"""Return a list of checkers, sorted by priority."""
return self._checkers
def register_checker(self, checker):
"""Register a checker instance."""
if checker not in self._checkers:
self._checkers.append(checker)
self.sort_checkers()
def unregister_checker(self, checker):
"""Unregister a checker instance."""
if checker in self._checkers:
self._checkers.remove(checker)
#-------------------------------------------------------------------------
# API for managing handlers
#-------------------------------------------------------------------------
def init_handlers(self):
"""Create the default handlers."""
self._handlers = {}
self._esc_handlers = {}
for handler in _default_handlers:
handler(
shell=self.shell, prefilter_manager=self, parent=self
)
@property
def handlers(self):
"""Return a dict of all the handlers."""
return self._handlers
def register_handler(self, name, handler, esc_strings):
"""Register a handler instance by name with esc_strings."""
self._handlers[name] = handler
for esc_str in esc_strings:
self._esc_handlers[esc_str] = handler
def unregister_handler(self, name, handler, esc_strings):
"""Unregister a handler instance by name with esc_strings."""
try:
del self._handlers[name]
except KeyError:
pass
for esc_str in esc_strings:
h = self._esc_handlers.get(esc_str)
if h is handler:
del self._esc_handlers[esc_str]
def get_handler_by_name(self, name):
"""Get a handler by its name."""
return self._handlers.get(name)
def get_handler_by_esc(self, esc_str):
"""Get a handler by its escape string."""
return self._esc_handlers.get(esc_str)
#-------------------------------------------------------------------------
# Main prefiltering API
#-------------------------------------------------------------------------
def prefilter_line_info(self, line_info):
"""Prefilter a line that has been converted to a LineInfo object.
This implements the checker/handler part of the prefilter pipe.
"""
# print "prefilter_line_info: ", line_info
handler = self.find_handler(line_info)
return handler.handle(line_info)
def find_handler(self, line_info):
"""Find a handler for the line_info by trying checkers."""
for checker in self.checkers:
if checker.enabled:
handler = checker.check(line_info)
if handler:
return handler
return self.get_handler_by_name('normal')
def transform_line(self, line, continue_prompt):
"""Calls the enabled transformers in order of increasing priority."""
for transformer in self.transformers:
if transformer.enabled:
line = transformer.transform(line, continue_prompt)
return line
def prefilter_line(self, line, continue_prompt=False):
"""Prefilter a single input line as text.
This method prefilters a single line of text by calling the
transformers and then the checkers/handlers.
"""
# print "prefilter_line: ", line, continue_prompt
# All handlers *must* return a value, even if it's blank ('').
# save the line away in case we crash, so the post-mortem handler can
# record it
self.shell._last_input_line = line
if not line:
# Return immediately on purely empty lines, so that if the user
# previously typed some whitespace that started a continuation
# prompt, he can break out of that loop with just an empty line.
# This is how the default python prompt works.
return ''
# At this point, we invoke our transformers.
if not continue_prompt or (continue_prompt and self.multi_line_specials):
line = self.transform_line(line, continue_prompt)
# Now we compute line_info for the checkers and handlers
line_info = LineInfo(line, continue_prompt)
# the input history needs to track even empty lines
stripped = line.strip()
normal_handler = self.get_handler_by_name('normal')
if not stripped:
return normal_handler.handle(line_info)
# special handlers are only allowed for single line statements
if continue_prompt and not self.multi_line_specials:
return normal_handler.handle(line_info)
prefiltered = self.prefilter_line_info(line_info)
# print "prefiltered line: %r" % prefiltered
return prefiltered
def prefilter_lines(self, lines, continue_prompt=False):
"""Prefilter multiple input lines of text.
This is the main entry point for prefiltering multiple lines of
input. This simply calls :meth:`prefilter_line` for each line of
input.
This covers cases where there are multiple lines in the user entry,
which is the case when the user goes back to a multiline history
entry and presses enter.
"""
llines = lines.rstrip('\n').split('\n')
# We can get multiple lines in one shot, where multiline input 'blends'
# into one line, in cases like recalling from the readline history
# buffer. We need to make sure that in such cases, we correctly
# communicate downstream which line is first and which are continuation
# ones.
if len(llines) > 1:
out = '\n'.join([self.prefilter_line(line, lnum>0)
for lnum, line in enumerate(llines) ])
else:
out = self.prefilter_line(llines[0], continue_prompt)
return out
#-----------------------------------------------------------------------------
# Prefilter transformers
#-----------------------------------------------------------------------------
class PrefilterTransformer(Configurable):
"""Transform a line of user input."""
priority = Integer(100).tag(config=True)
# Transformers don't currently use shell or prefilter_manager, but as we
# move away from checkers and handlers, they will need them.
shell = Instance('IPython.core.interactiveshell.InteractiveShellABC', allow_none=True)
prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager', allow_none=True)
enabled = Bool(True).tag(config=True)
def __init__(self, shell=None, prefilter_manager=None, **kwargs):
super(PrefilterTransformer, self).__init__(
shell=shell, prefilter_manager=prefilter_manager, **kwargs
)
self.prefilter_manager.register_transformer(self)
def transform(self, line, continue_prompt):
"""Transform a line, returning the new one."""
return None
def __repr__(self):
return "<%s(priority=%r, enabled=%r)>" % (
self.__class__.__name__, self.priority, self.enabled)
#-----------------------------------------------------------------------------
# Prefilter checkers
#-----------------------------------------------------------------------------
class PrefilterChecker(Configurable):
"""Inspect an input line and return a handler for that line."""
priority = Integer(100).tag(config=True)
shell = Instance('IPython.core.interactiveshell.InteractiveShellABC', allow_none=True)
prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager', allow_none=True)
enabled = Bool(True).tag(config=True)
def __init__(self, shell=None, prefilter_manager=None, **kwargs):
super(PrefilterChecker, self).__init__(
shell=shell, prefilter_manager=prefilter_manager, **kwargs
)
self.prefilter_manager.register_checker(self)
def check(self, line_info):
"""Inspect line_info and return a handler instance or None."""
return None
def __repr__(self):
return "<%s(priority=%r, enabled=%r)>" % (
self.__class__.__name__, self.priority, self.enabled)
class EmacsChecker(PrefilterChecker):
priority = Integer(100).tag(config=True)
enabled = Bool(False).tag(config=True)
def check(self, line_info):
"Emacs ipython-mode tags certain input lines."
if line_info.line.endswith('# PYTHON-MODE'):
return self.prefilter_manager.get_handler_by_name('emacs')
else:
return None
class MacroChecker(PrefilterChecker):
priority = Integer(250).tag(config=True)
def check(self, line_info):
obj = self.shell.user_ns.get(line_info.ifun)
if isinstance(obj, Macro):
return self.prefilter_manager.get_handler_by_name('macro')
else:
return None
class IPyAutocallChecker(PrefilterChecker):
priority = Integer(300).tag(config=True)
def check(self, line_info):
"Instances of IPyAutocall in user_ns get autocalled immediately"
obj = self.shell.user_ns.get(line_info.ifun, None)
if isinstance(obj, IPyAutocall):
obj.set_ip(self.shell)
return self.prefilter_manager.get_handler_by_name('auto')
else:
return None
class AssignmentChecker(PrefilterChecker):
priority = Integer(600).tag(config=True)
def check(self, line_info):
"""Check to see if user is assigning to a var for the first time, in
which case we want to avoid any sort of automagic / autocall games.
This allows users to assign to either alias or magic names true python
variables (the magic/alias systems always take second seat to true
python code). E.g. ls='hi', or ls,that=1,2"""
if line_info.the_rest:
if line_info.the_rest[0] in '=,':
return self.prefilter_manager.get_handler_by_name('normal')
else:
return None
class AutoMagicChecker(PrefilterChecker):
priority = Integer(700).tag(config=True)
def check(self, line_info):
"""If the ifun is magic, and automagic is on, run it. Note: normal,
non-auto magic would already have been triggered via '%' in
check_esc_chars. This just checks for automagic. Also, before
triggering the magic handler, make sure that there is nothing in the
user namespace which could shadow it."""
if not self.shell.automagic or not self.shell.find_magic(line_info.ifun):
return None
# We have a likely magic method. Make sure we should actually call it.
if line_info.continue_prompt and not self.prefilter_manager.multi_line_specials:
return None
head = line_info.ifun.split('.',1)[0]
if is_shadowed(head, self.shell):
return None
return self.prefilter_manager.get_handler_by_name('magic')
class PythonOpsChecker(PrefilterChecker):
priority = Integer(900).tag(config=True)
def check(self, line_info):
"""If the 'rest' of the line begins with a function call or pretty much
any python operator, we should simply execute the line (regardless of
whether or not there's a possible autocall expansion). This avoids
spurious (and very confusing) geattr() accesses."""
if line_info.the_rest and line_info.the_rest[0] in '!=()<>,+*/%^&|':
return self.prefilter_manager.get_handler_by_name('normal')
else:
return None
class AutocallChecker(PrefilterChecker):
priority = Integer(1000).tag(config=True)
function_name_regexp = CRegExp(re_fun_name,
help="RegExp to identify potential function names."
).tag(config=True)
exclude_regexp = CRegExp(re_exclude_auto,
help="RegExp to exclude strings with this start from autocalling."
).tag(config=True)
def check(self, line_info):
"Check if the initial word/function is callable and autocall is on."
if not self.shell.autocall:
return None
oinfo = line_info.ofind(self.shell) # This can mutate state via getattr
if not oinfo['found']:
return None
ignored_funs = ['b', 'f', 'r', 'u', 'br', 'rb', 'fr', 'rf']
ifun = line_info.ifun
line = line_info.line
if ifun.lower() in ignored_funs and (line.startswith(ifun + "'") or line.startswith(ifun + '"')):
return None
if callable(oinfo['obj']) \
and (not self.exclude_regexp.match(line_info.the_rest)) \
and self.function_name_regexp.match(line_info.ifun):
return self.prefilter_manager.get_handler_by_name('auto')
else:
return None
#-----------------------------------------------------------------------------
# Prefilter handlers
#-----------------------------------------------------------------------------
class PrefilterHandler(Configurable):
handler_name = Unicode('normal')
esc_strings = List([])
shell = Instance('IPython.core.interactiveshell.InteractiveShellABC', allow_none=True)
prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager', allow_none=True)
def __init__(self, shell=None, prefilter_manager=None, **kwargs):
super(PrefilterHandler, self).__init__(
shell=shell, prefilter_manager=prefilter_manager, **kwargs
)
self.prefilter_manager.register_handler(
self.handler_name,
self,
self.esc_strings
)
def handle(self, line_info):
# print "normal: ", line_info
"""Handle normal input lines. Use as a template for handlers."""
# With autoindent on, we need some way to exit the input loop, and I
# don't want to force the user to have to backspace all the way to
# clear the line. The rule will be in this case, that either two
# lines of pure whitespace in a row, or a line of pure whitespace but
# of a size different to the indent level, will exit the input loop.
line = line_info.line
continue_prompt = line_info.continue_prompt
if (continue_prompt and
self.shell.autoindent and
line.isspace() and
0 < abs(len(line) - self.shell.indent_current_nsp) <= 2):
line = ''
return line
def __str__(self):
return "<%s(name=%s)>" % (self.__class__.__name__, self.handler_name)
class MacroHandler(PrefilterHandler):
handler_name = Unicode("macro")
def handle(self, line_info):
obj = self.shell.user_ns.get(line_info.ifun)
pre_space = line_info.pre_whitespace
line_sep = "\n" + pre_space
return pre_space + line_sep.join(obj.value.splitlines())
class MagicHandler(PrefilterHandler):
handler_name = Unicode('magic')
esc_strings = List([ESC_MAGIC])
def handle(self, line_info):
"""Execute magic functions."""
ifun = line_info.ifun
the_rest = line_info.the_rest
#Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
t_arg_s = ifun + " " + the_rest
t_magic_name, _, t_magic_arg_s = t_arg_s.partition(' ')
t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
cmd = '%sget_ipython().run_line_magic(%r, %r)' % (line_info.pre_whitespace, t_magic_name, t_magic_arg_s)
return cmd
class AutoHandler(PrefilterHandler):
handler_name = Unicode('auto')
esc_strings = List([ESC_PAREN, ESC_QUOTE, ESC_QUOTE2])
def handle(self, line_info):
"""Handle lines which can be auto-executed, quoting if requested."""
line = line_info.line
ifun = line_info.ifun
the_rest = line_info.the_rest
esc = line_info.esc
continue_prompt = line_info.continue_prompt
obj = line_info.ofind(self.shell)['obj']
# This should only be active for single-line input!
if continue_prompt:
return line
force_auto = isinstance(obj, IPyAutocall)
# User objects sometimes raise exceptions on attribute access other
# than AttributeError (we've seen it in the past), so it's safest to be
# ultra-conservative here and catch all.
try:
auto_rewrite = obj.rewrite
except Exception:
auto_rewrite = True
if esc == ESC_QUOTE:
# Auto-quote splitting on whitespace
newcmd = '%s("%s")' % (ifun,'", "'.join(the_rest.split()) )
elif esc == ESC_QUOTE2:
# Auto-quote whole string
newcmd = '%s("%s")' % (ifun,the_rest)
elif esc == ESC_PAREN:
newcmd = '%s(%s)' % (ifun,",".join(the_rest.split()))
else:
# Auto-paren.
if force_auto:
# Don't rewrite if it is already a call.
do_rewrite = not the_rest.startswith('(')
else:
if not the_rest:
# We only apply it to argument-less calls if the autocall
# parameter is set to 2.
do_rewrite = (self.shell.autocall >= 2)
elif the_rest.startswith('[') and hasattr(obj, '__getitem__'):
# Don't autocall in this case: item access for an object
# which is BOTH callable and implements __getitem__.
do_rewrite = False
else:
do_rewrite = True
# Figure out the rewritten command
if do_rewrite:
if the_rest.endswith(';'):
newcmd = '%s(%s);' % (ifun.rstrip(),the_rest[:-1])
else:
newcmd = '%s(%s)' % (ifun.rstrip(), the_rest)
else:
normal_handler = self.prefilter_manager.get_handler_by_name('normal')
return normal_handler.handle(line_info)
# Display the rewritten call
if auto_rewrite:
self.shell.auto_rewrite_input(newcmd)
return newcmd
class EmacsHandler(PrefilterHandler):
handler_name = Unicode('emacs')
esc_strings = List([])
def handle(self, line_info):
"""Handle input lines marked by python-mode."""
# Currently, nothing is done. Later more functionality can be added
# here if needed.
# The input cache shouldn't be updated
return line_info.line
#-----------------------------------------------------------------------------
# Defaults
#-----------------------------------------------------------------------------
_default_checkers = [
EmacsChecker,
MacroChecker,
IPyAutocallChecker,
AssignmentChecker,
AutoMagicChecker,
PythonOpsChecker,
AutocallChecker
]
_default_handlers = [
PrefilterHandler,
MacroHandler,
MagicHandler,
AutoHandler,
EmacsHandler
]

@ -0,0 +1,11 @@
This is the IPython startup directory
.py and .ipy files in this directory will be run *prior* to any code or files specified
via the exec_lines or exec_files configurables whenever you load this profile.
Files will be run in lexicographical order, so you can control the execution order of files
with a prefix, e.g.::
00-first.py
50-middle.py
99-last.ipy

@ -0,0 +1,312 @@
# encoding: utf-8
"""
An application for managing IPython profiles.
To be invoked as the `ipython profile` subcommand.
Authors:
* Min RK
"""
#-----------------------------------------------------------------------------
# Copyright (C) 2008 The IPython Development Team
#
# Distributed under the terms of the BSD License. The full license is in
# the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------
#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
import os
from traitlets.config.application import Application
from IPython.core.application import (
BaseIPythonApplication, base_flags
)
from IPython.core.profiledir import ProfileDir
from IPython.utils.importstring import import_item
from IPython.paths import get_ipython_dir, get_ipython_package_dir
from traitlets import Unicode, Bool, Dict, observe
#-----------------------------------------------------------------------------
# Constants
#-----------------------------------------------------------------------------
create_help = """Create an IPython profile by name
Create an ipython profile directory by its name or
profile directory path. Profile directories contain
configuration, log and security related files and are named
using the convention 'profile_<name>'. By default they are
located in your ipython directory. Once created, you will
can edit the configuration files in the profile
directory to configure IPython. Most users will create a
profile directory by name,
`ipython profile create myprofile`, which will put the directory
in `<ipython_dir>/profile_myprofile`.
"""
list_help = """List available IPython profiles
List all available profiles, by profile location, that can
be found in the current working directly or in the ipython
directory. Profile directories are named using the convention
'profile_<profile>'.
"""
profile_help = """Manage IPython profiles
Profile directories contain
configuration, log and security related files and are named
using the convention 'profile_<name>'. By default they are
located in your ipython directory. You can create profiles
with `ipython profile create <name>`, or see the profiles you
already have with `ipython profile list`
To get started configuring IPython, simply do:
$> ipython profile create
and IPython will create the default profile in <ipython_dir>/profile_default,
where you can edit ipython_config.py to start configuring IPython.
"""
_list_examples = "ipython profile list # list all profiles"
_create_examples = """
ipython profile create foo # create profile foo w/ default config files
ipython profile create foo --reset # restage default config files over current
ipython profile create foo --parallel # also stage parallel config files
"""
_main_examples = """
ipython profile create -h # show the help string for the create subcommand
ipython profile list -h # show the help string for the list subcommand
ipython locate profile foo # print the path to the directory for profile 'foo'
"""
#-----------------------------------------------------------------------------
# Profile Application Class (for `ipython profile` subcommand)
#-----------------------------------------------------------------------------
def list_profiles_in(path):
"""list profiles in a given root directory"""
profiles = []
# for python 3.6+ rewrite to: with os.scandir(path) as dirlist:
files = os.scandir(path)
for f in files:
if f.is_dir() and f.name.startswith('profile_'):
profiles.append(f.name.split('_', 1)[-1])
return profiles
def list_bundled_profiles():
"""list profiles that are bundled with IPython."""
path = os.path.join(get_ipython_package_dir(), u'core', u'profile')
profiles = []
# for python 3.6+ rewrite to: with os.scandir(path) as dirlist:
files = os.scandir(path)
for profile in files:
if profile.is_dir() and profile.name != "__pycache__":
profiles.append(profile.name)
return profiles
class ProfileLocate(BaseIPythonApplication):
description = """print the path to an IPython profile dir"""
def parse_command_line(self, argv=None):
super(ProfileLocate, self).parse_command_line(argv)
if self.extra_args:
self.profile = self.extra_args[0]
def start(self):
print(self.profile_dir.location)
class ProfileList(Application):
name = u'ipython-profile'
description = list_help
examples = _list_examples
aliases = Dict({
'ipython-dir' : 'ProfileList.ipython_dir',
'log-level' : 'Application.log_level',
})
flags = Dict(dict(
debug = ({'Application' : {'log_level' : 0}},
"Set Application.log_level to 0, maximizing log output."
)
))
ipython_dir = Unicode(get_ipython_dir(),
help="""
The name of the IPython directory. This directory is used for logging
configuration (through profiles), history storage, etc. The default
is usually $HOME/.ipython. This options can also be specified through
the environment variable IPYTHONDIR.
"""
).tag(config=True)
def _print_profiles(self, profiles):
"""print list of profiles, indented."""
for profile in profiles:
print(' %s' % profile)
def list_profile_dirs(self):
profiles = list_bundled_profiles()
if profiles:
print()
print("Available profiles in IPython:")
self._print_profiles(profiles)
print()
print(" The first request for a bundled profile will copy it")
print(" into your IPython directory (%s)," % self.ipython_dir)
print(" where you can customize it.")
profiles = list_profiles_in(self.ipython_dir)
if profiles:
print()
print("Available profiles in %s:" % self.ipython_dir)
self._print_profiles(profiles)
profiles = list_profiles_in(os.getcwd())
if profiles:
print()
print(
"Profiles from CWD have been removed for security reason, see CVE-2022-21699:"
)
print()
print("To use any of the above profiles, start IPython with:")
print(" ipython --profile=<name>")
print()
def start(self):
self.list_profile_dirs()
create_flags = {}
create_flags.update(base_flags)
# don't include '--init' flag, which implies running profile create in other apps
create_flags.pop('init')
create_flags['reset'] = ({'ProfileCreate': {'overwrite' : True}},
"reset config files in this profile to the defaults.")
create_flags['parallel'] = ({'ProfileCreate': {'parallel' : True}},
"Include the config files for parallel "
"computing apps (ipengine, ipcontroller, etc.)")
class ProfileCreate(BaseIPythonApplication):
name = u'ipython-profile'
description = create_help
examples = _create_examples
auto_create = Bool(True)
def _log_format_default(self):
return "[%(name)s] %(message)s"
def _copy_config_files_default(self):
return True
parallel = Bool(False,
help="whether to include parallel computing config files"
).tag(config=True)
@observe('parallel')
def _parallel_changed(self, change):
parallel_files = [ 'ipcontroller_config.py',
'ipengine_config.py',
'ipcluster_config.py'
]
if change['new']:
for cf in parallel_files:
self.config_files.append(cf)
else:
for cf in parallel_files:
if cf in self.config_files:
self.config_files.remove(cf)
def parse_command_line(self, argv):
super(ProfileCreate, self).parse_command_line(argv)
# accept positional arg as profile name
if self.extra_args:
self.profile = self.extra_args[0]
flags = Dict(create_flags)
classes = [ProfileDir]
def _import_app(self, app_path):
"""import an app class"""
app = None
name = app_path.rsplit('.', 1)[-1]
try:
app = import_item(app_path)
except ImportError:
self.log.info("Couldn't import %s, config file will be excluded", name)
except Exception:
self.log.warning('Unexpected error importing %s', name, exc_info=True)
return app
def init_config_files(self):
super(ProfileCreate, self).init_config_files()
# use local imports, since these classes may import from here
from IPython.terminal.ipapp import TerminalIPythonApp
apps = [TerminalIPythonApp]
for app_path in (
'ipykernel.kernelapp.IPKernelApp',
):
app = self._import_app(app_path)
if app is not None:
apps.append(app)
if self.parallel:
from ipyparallel.apps.ipcontrollerapp import IPControllerApp
from ipyparallel.apps.ipengineapp import IPEngineApp
from ipyparallel.apps.ipclusterapp import IPClusterStart
apps.extend([
IPControllerApp,
IPEngineApp,
IPClusterStart,
])
for App in apps:
app = App()
app.config.update(self.config)
app.log = self.log
app.overwrite = self.overwrite
app.copy_config_files=True
app.ipython_dir=self.ipython_dir
app.profile_dir=self.profile_dir
app.init_config_files()
def stage_default_config_file(self):
pass
class ProfileApp(Application):
name = u'ipython profile'
description = profile_help
examples = _main_examples
subcommands = Dict(dict(
create = (ProfileCreate, ProfileCreate.description.splitlines()[0]),
list = (ProfileList, ProfileList.description.splitlines()[0]),
locate = (ProfileLocate, ProfileLocate.description.splitlines()[0]),
))
def start(self):
if self.subapp is None:
print("No subcommand specified. Must specify one of: %s"%(self.subcommands.keys()))
print()
self.print_description()
self.print_subcommands()
self.exit(1)
else:
return self.subapp.start()

@ -0,0 +1,225 @@
# encoding: utf-8
"""An object for managing IPython profile directories."""
# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.
import os
import shutil
import errno
from pathlib import Path
from traitlets.config.configurable import LoggingConfigurable
from ..paths import get_ipython_package_dir
from ..utils.path import expand_path, ensure_dir_exists
from traitlets import Unicode, Bool, observe
#-----------------------------------------------------------------------------
# Module errors
#-----------------------------------------------------------------------------
class ProfileDirError(Exception):
pass
#-----------------------------------------------------------------------------
# Class for managing profile directories
#-----------------------------------------------------------------------------
class ProfileDir(LoggingConfigurable):
"""An object to manage the profile directory and its resources.
The profile directory is used by all IPython applications, to manage
configuration, logging and security.
This object knows how to find, create and manage these directories. This
should be used by any code that wants to handle profiles.
"""
security_dir_name = Unicode('security')
log_dir_name = Unicode('log')
startup_dir_name = Unicode('startup')
pid_dir_name = Unicode('pid')
static_dir_name = Unicode('static')
security_dir = Unicode(u'')
log_dir = Unicode(u'')
startup_dir = Unicode(u'')
pid_dir = Unicode(u'')
static_dir = Unicode(u'')
location = Unicode(u'',
help="""Set the profile location directly. This overrides the logic used by the
`profile` option.""",
).tag(config=True)
_location_isset = Bool(False) # flag for detecting multiply set location
@observe('location')
def _location_changed(self, change):
if self._location_isset:
raise RuntimeError("Cannot set profile location more than once.")
self._location_isset = True
new = change['new']
ensure_dir_exists(new)
# ensure config files exist:
self.security_dir = os.path.join(new, self.security_dir_name)
self.log_dir = os.path.join(new, self.log_dir_name)
self.startup_dir = os.path.join(new, self.startup_dir_name)
self.pid_dir = os.path.join(new, self.pid_dir_name)
self.static_dir = os.path.join(new, self.static_dir_name)
self.check_dirs()
def _mkdir(self, path, mode=None):
"""ensure a directory exists at a given path
This is a version of os.mkdir, with the following differences:
- returns True if it created the directory, False otherwise
- ignores EEXIST, protecting against race conditions where
the dir may have been created in between the check and
the creation
- sets permissions if requested and the dir already exists
"""
if os.path.exists(path):
if mode and os.stat(path).st_mode != mode:
try:
os.chmod(path, mode)
except OSError:
self.log.warning(
"Could not set permissions on %s",
path
)
return False
try:
if mode:
os.mkdir(path, mode)
else:
os.mkdir(path)
except OSError as e:
if e.errno == errno.EEXIST:
return False
else:
raise
return True
@observe('log_dir')
def check_log_dir(self, change=None):
self._mkdir(self.log_dir)
@observe('startup_dir')
def check_startup_dir(self, change=None):
self._mkdir(self.startup_dir)
readme = os.path.join(self.startup_dir, 'README')
src = os.path.join(get_ipython_package_dir(), u'core', u'profile', u'README_STARTUP')
if not os.path.exists(src):
self.log.warning("Could not copy README_STARTUP to startup dir. Source file %s does not exist.", src)
if os.path.exists(src) and not os.path.exists(readme):
shutil.copy(src, readme)
@observe('security_dir')
def check_security_dir(self, change=None):
self._mkdir(self.security_dir, 0o40700)
@observe('pid_dir')
def check_pid_dir(self, change=None):
self._mkdir(self.pid_dir, 0o40700)
def check_dirs(self):
self.check_security_dir()
self.check_log_dir()
self.check_pid_dir()
self.check_startup_dir()
def copy_config_file(self, config_file: str, path: Path, overwrite=False) -> bool:
"""Copy a default config file into the active profile directory.
Default configuration files are kept in :mod:`IPython.core.profile`.
This function moves these from that location to the working profile
directory.
"""
dst = Path(os.path.join(self.location, config_file))
if dst.exists() and not overwrite:
return False
if path is None:
path = os.path.join(get_ipython_package_dir(), u'core', u'profile', u'default')
assert isinstance(path, Path)
src = path / config_file
shutil.copy(src, dst)
return True
@classmethod
def create_profile_dir(cls, profile_dir, config=None):
"""Create a new profile directory given a full path.
Parameters
----------
profile_dir : str
The full path to the profile directory. If it does exist, it will
be used. If not, it will be created.
"""
return cls(location=profile_dir, config=config)
@classmethod
def create_profile_dir_by_name(cls, path, name=u'default', config=None):
"""Create a profile dir by profile name and path.
Parameters
----------
path : unicode
The path (directory) to put the profile directory in.
name : unicode
The name of the profile. The name of the profile directory will
be "profile_<profile>".
"""
if not os.path.isdir(path):
raise ProfileDirError('Directory not found: %s' % path)
profile_dir = os.path.join(path, u'profile_' + name)
return cls(location=profile_dir, config=config)
@classmethod
def find_profile_dir_by_name(cls, ipython_dir, name=u'default', config=None):
"""Find an existing profile dir by profile name, return its ProfileDir.
This searches through a sequence of paths for a profile dir. If it
is not found, a :class:`ProfileDirError` exception will be raised.
The search path algorithm is:
1. ``os.getcwd()`` # removed for security reason.
2. ``ipython_dir``
Parameters
----------
ipython_dir : unicode or str
The IPython directory to use.
name : unicode or str
The name of the profile. The name of the profile directory
will be "profile_<profile>".
"""
dirname = u'profile_' + name
paths = [ipython_dir]
for p in paths:
profile_dir = os.path.join(p, dirname)
if os.path.isdir(profile_dir):
return cls(location=profile_dir, config=config)
else:
raise ProfileDirError('Profile directory not found in paths: %s' % dirname)
@classmethod
def find_profile_dir(cls, profile_dir, config=None):
"""Find/create a profile dir and return its ProfileDir.
This will create the profile directory if it doesn't exist.
Parameters
----------
profile_dir : unicode or str
The path of the profile directory.
"""
profile_dir = expand_path(profile_dir)
if not os.path.isdir(profile_dir):
raise ProfileDirError('Profile directory not found: %s' % profile_dir)
return cls(location=profile_dir, config=config)

@ -0,0 +1,21 @@
# -*- coding: utf-8 -*-
"""Being removed
"""
class LazyEvaluate(object):
"""This is used for formatting strings with values that need to be updated
at that time, such as the current time or working directory."""
def __init__(self, func, *args, **kwargs):
self.func = func
self.args = args
self.kwargs = kwargs
def __call__(self, **kwargs):
self.kwargs.update(kwargs)
return self.func(*self.args, **self.kwargs)
def __str__(self):
return str(self())
def __format__(self, format_spec):
return format(self(), format_spec)

@ -0,0 +1,424 @@
# -*- coding: utf-8 -*-
"""Pylab (matplotlib) support utilities."""
# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.
from io import BytesIO
from binascii import b2a_base64
from functools import partial
import warnings
from IPython.core.display import _pngxy
from IPython.utils.decorators import flag_calls
# If user specifies a GUI, that dictates the backend, otherwise we read the
# user's mpl default from the mpl rc structure
backends = {
"tk": "TkAgg",
"gtk": "GTKAgg",
"gtk3": "GTK3Agg",
"gtk4": "GTK4Agg",
"wx": "WXAgg",
"qt4": "Qt4Agg",
"qt5": "Qt5Agg",
"qt6": "QtAgg",
"qt": "Qt5Agg",
"osx": "MacOSX",
"nbagg": "nbAgg",
"notebook": "nbAgg",
"agg": "agg",
"svg": "svg",
"pdf": "pdf",
"ps": "ps",
"inline": "module://matplotlib_inline.backend_inline",
"ipympl": "module://ipympl.backend_nbagg",
"widget": "module://ipympl.backend_nbagg",
}
# We also need a reverse backends2guis mapping that will properly choose which
# GUI support to activate based on the desired matplotlib backend. For the
# most part it's just a reverse of the above dict, but we also need to add a
# few others that map to the same GUI manually:
backend2gui = dict(zip(backends.values(), backends.keys()))
# In the reverse mapping, there are a few extra valid matplotlib backends that
# map to the same GUI support
backend2gui["GTK"] = backend2gui["GTKCairo"] = "gtk"
backend2gui["GTK3Cairo"] = "gtk3"
backend2gui["GTK4Cairo"] = "gtk4"
backend2gui["WX"] = "wx"
backend2gui["CocoaAgg"] = "osx"
# There needs to be a hysteresis here as the new QtAgg Matplotlib backend
# supports either Qt5 or Qt6 and the IPython qt event loop support Qt4, Qt5,
# and Qt6.
backend2gui["QtAgg"] = "qt"
backend2gui["Qt4Agg"] = "qt"
backend2gui["Qt5Agg"] = "qt"
# And some backends that don't need GUI integration
del backend2gui["nbAgg"]
del backend2gui["agg"]
del backend2gui["svg"]
del backend2gui["pdf"]
del backend2gui["ps"]
del backend2gui["module://matplotlib_inline.backend_inline"]
del backend2gui["module://ipympl.backend_nbagg"]
#-----------------------------------------------------------------------------
# Matplotlib utilities
#-----------------------------------------------------------------------------
def getfigs(*fig_nums):
"""Get a list of matplotlib figures by figure numbers.
If no arguments are given, all available figures are returned. If the
argument list contains references to invalid figures, a warning is printed
but the function continues pasting further figures.
Parameters
----------
figs : tuple
A tuple of ints giving the figure numbers of the figures to return.
"""
from matplotlib._pylab_helpers import Gcf
if not fig_nums:
fig_managers = Gcf.get_all_fig_managers()
return [fm.canvas.figure for fm in fig_managers]
else:
figs = []
for num in fig_nums:
f = Gcf.figs.get(num)
if f is None:
print('Warning: figure %s not available.' % num)
else:
figs.append(f.canvas.figure)
return figs
def figsize(sizex, sizey):
"""Set the default figure size to be [sizex, sizey].
This is just an easy to remember, convenience wrapper that sets::
matplotlib.rcParams['figure.figsize'] = [sizex, sizey]
"""
import matplotlib
matplotlib.rcParams['figure.figsize'] = [sizex, sizey]
def print_figure(fig, fmt="png", bbox_inches="tight", base64=False, **kwargs):
"""Print a figure to an image, and return the resulting file data
Returned data will be bytes unless ``fmt='svg'``,
in which case it will be unicode.
Any keyword args are passed to fig.canvas.print_figure,
such as ``quality`` or ``bbox_inches``.
If `base64` is True, return base64-encoded str instead of raw bytes
for binary-encoded image formats
.. versionadded:: 7.29
base64 argument
"""
# When there's an empty figure, we shouldn't return anything, otherwise we
# get big blank areas in the qt console.
if not fig.axes and not fig.lines:
return
dpi = fig.dpi
if fmt == 'retina':
dpi = dpi * 2
fmt = 'png'
# build keyword args
kw = {
"format":fmt,
"facecolor":fig.get_facecolor(),
"edgecolor":fig.get_edgecolor(),
"dpi":dpi,
"bbox_inches":bbox_inches,
}
# **kwargs get higher priority
kw.update(kwargs)
bytes_io = BytesIO()
if fig.canvas is None:
from matplotlib.backend_bases import FigureCanvasBase
FigureCanvasBase(fig)
fig.canvas.print_figure(bytes_io, **kw)
data = bytes_io.getvalue()
if fmt == 'svg':
data = data.decode('utf-8')
elif base64:
data = b2a_base64(data).decode("ascii")
return data
def retina_figure(fig, base64=False, **kwargs):
"""format a figure as a pixel-doubled (retina) PNG
If `base64` is True, return base64-encoded str instead of raw bytes
for binary-encoded image formats
.. versionadded:: 7.29
base64 argument
"""
pngdata = print_figure(fig, fmt="retina", base64=False, **kwargs)
# Make sure that retina_figure acts just like print_figure and returns
# None when the figure is empty.
if pngdata is None:
return
w, h = _pngxy(pngdata)
metadata = {"width": w//2, "height":h//2}
if base64:
pngdata = b2a_base64(pngdata).decode("ascii")
return pngdata, metadata
# We need a little factory function here to create the closure where
# safe_execfile can live.
def mpl_runner(safe_execfile):
"""Factory to return a matplotlib-enabled runner for %run.
Parameters
----------
safe_execfile : function
This must be a function with the same interface as the
:meth:`safe_execfile` method of IPython.
Returns
-------
A function suitable for use as the ``runner`` argument of the %run magic
function.
"""
def mpl_execfile(fname,*where,**kw):
"""matplotlib-aware wrapper around safe_execfile.
Its interface is identical to that of the :func:`execfile` builtin.
This is ultimately a call to execfile(), but wrapped in safeties to
properly handle interactive rendering."""
import matplotlib
import matplotlib.pyplot as plt
#print '*** Matplotlib runner ***' # dbg
# turn off rendering until end of script
is_interactive = matplotlib.rcParams['interactive']
matplotlib.interactive(False)
safe_execfile(fname,*where,**kw)
matplotlib.interactive(is_interactive)
# make rendering call now, if the user tried to do it
if plt.draw_if_interactive.called:
plt.draw()
plt.draw_if_interactive.called = False
# re-draw everything that is stale
try:
da = plt.draw_all
except AttributeError:
pass
else:
da()
return mpl_execfile
def _reshow_nbagg_figure(fig):
"""reshow an nbagg figure"""
try:
reshow = fig.canvas.manager.reshow
except AttributeError as e:
raise NotImplementedError() from e
else:
reshow()
def select_figure_formats(shell, formats, **kwargs):
"""Select figure formats for the inline backend.
Parameters
----------
shell : InteractiveShell
The main IPython instance.
formats : str or set
One or a set of figure formats to enable: 'png', 'retina', 'jpeg', 'svg', 'pdf'.
**kwargs : any
Extra keyword arguments to be passed to fig.canvas.print_figure.
"""
import matplotlib
from matplotlib.figure import Figure
svg_formatter = shell.display_formatter.formatters['image/svg+xml']
png_formatter = shell.display_formatter.formatters['image/png']
jpg_formatter = shell.display_formatter.formatters['image/jpeg']
pdf_formatter = shell.display_formatter.formatters['application/pdf']
if isinstance(formats, str):
formats = {formats}
# cast in case of list / tuple
formats = set(formats)
[ f.pop(Figure, None) for f in shell.display_formatter.formatters.values() ]
mplbackend = matplotlib.get_backend().lower()
if mplbackend == 'nbagg' or mplbackend == 'module://ipympl.backend_nbagg':
formatter = shell.display_formatter.ipython_display_formatter
formatter.for_type(Figure, _reshow_nbagg_figure)
supported = {'png', 'png2x', 'retina', 'jpg', 'jpeg', 'svg', 'pdf'}
bad = formats.difference(supported)
if bad:
bs = "%s" % ','.join([repr(f) for f in bad])
gs = "%s" % ','.join([repr(f) for f in supported])
raise ValueError("supported formats are: %s not %s" % (gs, bs))
if "png" in formats:
png_formatter.for_type(
Figure, partial(print_figure, fmt="png", base64=True, **kwargs)
)
if "retina" in formats or "png2x" in formats:
png_formatter.for_type(Figure, partial(retina_figure, base64=True, **kwargs))
if "jpg" in formats or "jpeg" in formats:
jpg_formatter.for_type(
Figure, partial(print_figure, fmt="jpg", base64=True, **kwargs)
)
if "svg" in formats:
svg_formatter.for_type(Figure, partial(print_figure, fmt="svg", **kwargs))
if "pdf" in formats:
pdf_formatter.for_type(
Figure, partial(print_figure, fmt="pdf", base64=True, **kwargs)
)
#-----------------------------------------------------------------------------
# Code for initializing matplotlib and importing pylab
#-----------------------------------------------------------------------------
def find_gui_and_backend(gui=None, gui_select=None):
"""Given a gui string return the gui and mpl backend.
Parameters
----------
gui : str
Can be one of ('tk','gtk','wx','qt','qt4','inline','agg').
gui_select : str
Can be one of ('tk','gtk','wx','qt','qt4','inline').
This is any gui already selected by the shell.
Returns
-------
A tuple of (gui, backend) where backend is one of ('TkAgg','GTKAgg',
'WXAgg','Qt4Agg','module://matplotlib_inline.backend_inline','agg').
"""
import matplotlib
if gui and gui != 'auto':
# select backend based on requested gui
backend = backends[gui]
if gui == 'agg':
gui = None
else:
# We need to read the backend from the original data structure, *not*
# from mpl.rcParams, since a prior invocation of %matplotlib may have
# overwritten that.
# WARNING: this assumes matplotlib 1.1 or newer!!
backend = matplotlib.rcParamsOrig['backend']
# In this case, we need to find what the appropriate gui selection call
# should be for IPython, so we can activate inputhook accordingly
gui = backend2gui.get(backend, None)
# If we have already had a gui active, we need it and inline are the
# ones allowed.
if gui_select and gui != gui_select:
gui = gui_select
backend = backends[gui]
return gui, backend
def activate_matplotlib(backend):
"""Activate the given backend and set interactive to True."""
import matplotlib
matplotlib.interactive(True)
# Matplotlib had a bug where even switch_backend could not force
# the rcParam to update. This needs to be set *before* the module
# magic of switch_backend().
matplotlib.rcParams['backend'] = backend
# Due to circular imports, pyplot may be only partially initialised
# when this function runs.
# So avoid needing matplotlib attribute-lookup to access pyplot.
from matplotlib import pyplot as plt
plt.switch_backend(backend)
plt.show._needmain = False
# We need to detect at runtime whether show() is called by the user.
# For this, we wrap it into a decorator which adds a 'called' flag.
plt.draw_if_interactive = flag_calls(plt.draw_if_interactive)
def import_pylab(user_ns, import_all=True):
"""Populate the namespace with pylab-related values.
Imports matplotlib, pylab, numpy, and everything from pylab and numpy.
Also imports a few names from IPython (figsize, display, getfigs)
"""
# Import numpy as np/pyplot as plt are conventions we're trying to
# somewhat standardize on. Making them available to users by default
# will greatly help this.
s = ("import numpy\n"
"import matplotlib\n"
"from matplotlib import pylab, mlab, pyplot\n"
"np = numpy\n"
"plt = pyplot\n"
)
exec(s, user_ns)
if import_all:
s = ("from matplotlib.pylab import *\n"
"from numpy import *\n")
exec(s, user_ns)
# IPython symbols to add
user_ns['figsize'] = figsize
from IPython.display import display
# Add display and getfigs to the user's namespace
user_ns['display'] = display
user_ns['getfigs'] = getfigs
def configure_inline_support(shell, backend):
"""
.. deprecated:: 7.23
use `matplotlib_inline.backend_inline.configure_inline_support()`
Configure an IPython shell object for matplotlib use.
Parameters
----------
shell : InteractiveShell instance
backend : matplotlib backend
"""
warnings.warn(
"`configure_inline_support` is deprecated since IPython 7.23, directly "
"use `matplotlib_inline.backend_inline.configure_inline_support()`",
DeprecationWarning,
stacklevel=2,
)
from matplotlib_inline.backend_inline import (
configure_inline_support as configure_inline_support_orig,
)
configure_inline_support_orig(shell, backend)

@ -0,0 +1,54 @@
# -*- coding: utf-8 -*-
"""Release data for the IPython project."""
#-----------------------------------------------------------------------------
# Copyright (c) 2008, IPython Development Team.
# Copyright (c) 2001, Fernando Perez <fernando.perez@colorado.edu>
# Copyright (c) 2001, Janko Hauser <jhauser@zscout.de>
# Copyright (c) 2001, Nathaniel Gray <n8gray@caltech.edu>
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
# IPython version information. An empty _version_extra corresponds to a full
# release. 'dev' as a _version_extra string means this is a development
# version
_version_major = 8
_version_minor = 4
_version_patch = 0
_version_extra = ".dev"
# _version_extra = "rc1"
_version_extra = "" # Uncomment this for full releases
# Construct full version string from these.
_ver = [_version_major, _version_minor, _version_patch]
__version__ = '.'.join(map(str, _ver))
if _version_extra:
__version__ = __version__ + _version_extra
version = __version__ # backwards compatibility name
version_info = (_version_major, _version_minor, _version_patch, _version_extra)
# Change this when incrementing the kernel protocol version
kernel_protocol_version_info = (5, 0)
kernel_protocol_version = "%i.%i" % kernel_protocol_version_info
license = 'BSD'
authors = {'Fernando' : ('Fernando Perez','fperez.net@gmail.com'),
'Janko' : ('Janko Hauser','jhauser@zscout.de'),
'Nathan' : ('Nathaniel Gray','n8gray@caltech.edu'),
'Ville' : ('Ville Vainio','vivainio@gmail.com'),
'Brian' : ('Brian E Granger', 'ellisonbg@gmail.com'),
'Min' : ('Min Ragan-Kelley', 'benjaminrk@gmail.com'),
'Thomas' : ('Thomas A. Kluyver', 'takowl@gmail.com'),
'Jorgen' : ('Jorgen Stenarson', 'jorgen.stenarson@bostream.nu'),
'Matthias' : ('Matthias Bussonnier', 'bussonniermatthias@gmail.com'),
}
author = 'The IPython Development Team'
author_email = 'ipython-dev@python.org'

@ -0,0 +1,452 @@
# encoding: utf-8
"""
A mixin for :class:`~IPython.core.application.Application` classes that
launch InteractiveShell instances, load extensions, etc.
"""
# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.
import glob
from itertools import chain
import os
import sys
from traitlets.config.application import boolean_flag
from traitlets.config.configurable import Configurable
from traitlets.config.loader import Config
from IPython.core.application import SYSTEM_CONFIG_DIRS, ENV_CONFIG_DIRS
from IPython.core import pylabtools
from IPython.utils.contexts import preserve_keys
from IPython.utils.path import filefind
import traitlets
from traitlets import (
Unicode, Instance, List, Bool, CaselessStrEnum, observe,
DottedObjectName,
)
from IPython.terminal import pt_inputhooks
#-----------------------------------------------------------------------------
# Aliases and Flags
#-----------------------------------------------------------------------------
gui_keys = tuple(sorted(pt_inputhooks.backends) + sorted(pt_inputhooks.aliases))
backend_keys = sorted(pylabtools.backends.keys())
backend_keys.insert(0, 'auto')
shell_flags = {}
addflag = lambda *args: shell_flags.update(boolean_flag(*args))
addflag('autoindent', 'InteractiveShell.autoindent',
'Turn on autoindenting.', 'Turn off autoindenting.'
)
addflag('automagic', 'InteractiveShell.automagic',
"""Turn on the auto calling of magic commands. Type %%magic at the
IPython prompt for more information.""",
'Turn off the auto calling of magic commands.'
)
addflag('pdb', 'InteractiveShell.pdb',
"Enable auto calling the pdb debugger after every exception.",
"Disable auto calling the pdb debugger after every exception."
)
addflag('pprint', 'PlainTextFormatter.pprint',
"Enable auto pretty printing of results.",
"Disable auto pretty printing of results."
)
addflag('color-info', 'InteractiveShell.color_info',
"""IPython can display information about objects via a set of functions,
and optionally can use colors for this, syntax highlighting
source code and various other elements. This is on by default, but can cause
problems with some pagers. If you see such problems, you can disable the
colours.""",
"Disable using colors for info related things."
)
addflag('ignore-cwd', 'InteractiveShellApp.ignore_cwd',
"Exclude the current working directory from sys.path",
"Include the current working directory in sys.path",
)
nosep_config = Config()
nosep_config.InteractiveShell.separate_in = ''
nosep_config.InteractiveShell.separate_out = ''
nosep_config.InteractiveShell.separate_out2 = ''
shell_flags['nosep']=(nosep_config, "Eliminate all spacing between prompts.")
shell_flags['pylab'] = (
{'InteractiveShellApp' : {'pylab' : 'auto'}},
"""Pre-load matplotlib and numpy for interactive use with
the default matplotlib backend."""
)
shell_flags['matplotlib'] = (
{'InteractiveShellApp' : {'matplotlib' : 'auto'}},
"""Configure matplotlib for interactive use with
the default matplotlib backend."""
)
# it's possible we don't want short aliases for *all* of these:
shell_aliases = dict(
autocall='InteractiveShell.autocall',
colors='InteractiveShell.colors',
logfile='InteractiveShell.logfile',
logappend='InteractiveShell.logappend',
c='InteractiveShellApp.code_to_run',
m='InteractiveShellApp.module_to_run',
ext="InteractiveShellApp.extra_extensions",
gui='InteractiveShellApp.gui',
pylab='InteractiveShellApp.pylab',
matplotlib='InteractiveShellApp.matplotlib',
)
shell_aliases['cache-size'] = 'InteractiveShell.cache_size'
#-----------------------------------------------------------------------------
# Main classes and functions
#-----------------------------------------------------------------------------
class InteractiveShellApp(Configurable):
"""A Mixin for applications that start InteractiveShell instances.
Provides configurables for loading extensions and executing files
as part of configuring a Shell environment.
The following methods should be called by the :meth:`initialize` method
of the subclass:
- :meth:`init_path`
- :meth:`init_shell` (to be implemented by the subclass)
- :meth:`init_gui_pylab`
- :meth:`init_extensions`
- :meth:`init_code`
"""
extensions = List(Unicode(),
help="A list of dotted module names of IPython extensions to load."
).tag(config=True)
extra_extensions = List(
DottedObjectName(),
help="""
Dotted module name(s) of one or more IPython extensions to load.
For specifying extra extensions to load on the command-line.
.. versionadded:: 7.10
""",
).tag(config=True)
reraise_ipython_extension_failures = Bool(False,
help="Reraise exceptions encountered loading IPython extensions?",
).tag(config=True)
# Extensions that are always loaded (not configurable)
default_extensions = List(Unicode(), [u'storemagic']).tag(config=False)
hide_initial_ns = Bool(True,
help="""Should variables loaded at startup (by startup files, exec_lines, etc.)
be hidden from tools like %who?"""
).tag(config=True)
exec_files = List(Unicode(),
help="""List of files to run at IPython startup."""
).tag(config=True)
exec_PYTHONSTARTUP = Bool(True,
help="""Run the file referenced by the PYTHONSTARTUP environment
variable at IPython startup."""
).tag(config=True)
file_to_run = Unicode('',
help="""A file to be run""").tag(config=True)
exec_lines = List(Unicode(),
help="""lines of code to run at IPython startup."""
).tag(config=True)
code_to_run = Unicode('',
help="Execute the given command string."
).tag(config=True)
module_to_run = Unicode('',
help="Run the module as a script."
).tag(config=True)
gui = CaselessStrEnum(gui_keys, allow_none=True,
help="Enable GUI event loop integration with any of {0}.".format(gui_keys)
).tag(config=True)
matplotlib = CaselessStrEnum(backend_keys, allow_none=True,
help="""Configure matplotlib for interactive use with
the default matplotlib backend."""
).tag(config=True)
pylab = CaselessStrEnum(backend_keys, allow_none=True,
help="""Pre-load matplotlib and numpy for interactive use,
selecting a particular matplotlib backend and loop integration.
"""
).tag(config=True)
pylab_import_all = Bool(True,
help="""If true, IPython will populate the user namespace with numpy, pylab, etc.
and an ``import *`` is done from numpy and pylab, when using pylab mode.
When False, pylab mode should not import any names into the user namespace.
"""
).tag(config=True)
ignore_cwd = Bool(
False,
help="""If True, IPython will not add the current working directory to sys.path.
When False, the current working directory is added to sys.path, allowing imports
of modules defined in the current directory."""
).tag(config=True)
shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
allow_none=True)
# whether interact-loop should start
interact = Bool(True)
user_ns = Instance(dict, args=None, allow_none=True)
@observe('user_ns')
def _user_ns_changed(self, change):
if self.shell is not None:
self.shell.user_ns = change['new']
self.shell.init_user_ns()
def init_path(self):
"""Add current working directory, '', to sys.path
Unlike Python's default, we insert before the first `site-packages`
or `dist-packages` directory,
so that it is after the standard library.
.. versionchanged:: 7.2
Try to insert after the standard library, instead of first.
.. versionchanged:: 8.0
Allow optionally not including the current directory in sys.path
"""
if '' in sys.path or self.ignore_cwd:
return
for idx, path in enumerate(sys.path):
parent, last_part = os.path.split(path)
if last_part in {'site-packages', 'dist-packages'}:
break
else:
# no site-packages or dist-packages found (?!)
# back to original behavior of inserting at the front
idx = 0
sys.path.insert(idx, '')
def init_shell(self):
raise NotImplementedError("Override in subclasses")
def init_gui_pylab(self):
"""Enable GUI event loop integration, taking pylab into account."""
enable = False
shell = self.shell
if self.pylab:
enable = lambda key: shell.enable_pylab(key, import_all=self.pylab_import_all)
key = self.pylab
elif self.matplotlib:
enable = shell.enable_matplotlib
key = self.matplotlib
elif self.gui:
enable = shell.enable_gui
key = self.gui
if not enable:
return
try:
r = enable(key)
except ImportError:
self.log.warning("Eventloop or matplotlib integration failed. Is matplotlib installed?")
self.shell.showtraceback()
return
except Exception:
self.log.warning("GUI event loop or pylab initialization failed")
self.shell.showtraceback()
return
if isinstance(r, tuple):
gui, backend = r[:2]
self.log.info("Enabling GUI event loop integration, "
"eventloop=%s, matplotlib=%s", gui, backend)
if key == "auto":
print("Using matplotlib backend: %s" % backend)
else:
gui = r
self.log.info("Enabling GUI event loop integration, "
"eventloop=%s", gui)
def init_extensions(self):
"""Load all IPython extensions in IPythonApp.extensions.
This uses the :meth:`ExtensionManager.load_extensions` to load all
the extensions listed in ``self.extensions``.
"""
try:
self.log.debug("Loading IPython extensions...")
extensions = (
self.default_extensions + self.extensions + self.extra_extensions
)
for ext in extensions:
try:
self.log.info("Loading IPython extension: %s" % ext)
self.shell.extension_manager.load_extension(ext)
except:
if self.reraise_ipython_extension_failures:
raise
msg = ("Error in loading extension: {ext}\n"
"Check your config files in {location}".format(
ext=ext,
location=self.profile_dir.location
))
self.log.warning(msg, exc_info=True)
except:
if self.reraise_ipython_extension_failures:
raise
self.log.warning("Unknown error in loading extensions:", exc_info=True)
def init_code(self):
"""run the pre-flight code, specified via exec_lines"""
self._run_startup_files()
self._run_exec_lines()
self._run_exec_files()
# Hide variables defined here from %who etc.
if self.hide_initial_ns:
self.shell.user_ns_hidden.update(self.shell.user_ns)
# command-line execution (ipython -i script.py, ipython -m module)
# should *not* be excluded from %whos
self._run_cmd_line_code()
self._run_module()
# flush output, so itwon't be attached to the first cell
sys.stdout.flush()
sys.stderr.flush()
self.shell._sys_modules_keys = set(sys.modules.keys())
def _run_exec_lines(self):
"""Run lines of code in IPythonApp.exec_lines in the user's namespace."""
if not self.exec_lines:
return
try:
self.log.debug("Running code from IPythonApp.exec_lines...")
for line in self.exec_lines:
try:
self.log.info("Running code in user namespace: %s" %
line)
self.shell.run_cell(line, store_history=False)
except:
self.log.warning("Error in executing line in user "
"namespace: %s" % line)
self.shell.showtraceback()
except:
self.log.warning("Unknown error in handling IPythonApp.exec_lines:")
self.shell.showtraceback()
def _exec_file(self, fname, shell_futures=False):
try:
full_filename = filefind(fname, [u'.', self.ipython_dir])
except IOError:
self.log.warning("File not found: %r"%fname)
return
# Make sure that the running script gets a proper sys.argv as if it
# were run from a system shell.
save_argv = sys.argv
sys.argv = [full_filename] + self.extra_args[1:]
try:
if os.path.isfile(full_filename):
self.log.info("Running file in user namespace: %s" %
full_filename)
# Ensure that __file__ is always defined to match Python
# behavior.
with preserve_keys(self.shell.user_ns, '__file__'):
self.shell.user_ns['__file__'] = fname
if full_filename.endswith('.ipy') or full_filename.endswith('.ipynb'):
self.shell.safe_execfile_ipy(full_filename,
shell_futures=shell_futures)
else:
# default to python, even without extension
self.shell.safe_execfile(full_filename,
self.shell.user_ns,
shell_futures=shell_futures,
raise_exceptions=True)
finally:
sys.argv = save_argv
def _run_startup_files(self):
"""Run files from profile startup directory"""
startup_dirs = [self.profile_dir.startup_dir] + [
os.path.join(p, 'startup') for p in chain(ENV_CONFIG_DIRS, SYSTEM_CONFIG_DIRS)
]
startup_files = []
if self.exec_PYTHONSTARTUP and os.environ.get('PYTHONSTARTUP', False) and \
not (self.file_to_run or self.code_to_run or self.module_to_run):
python_startup = os.environ['PYTHONSTARTUP']
self.log.debug("Running PYTHONSTARTUP file %s...", python_startup)
try:
self._exec_file(python_startup)
except:
self.log.warning("Unknown error in handling PYTHONSTARTUP file %s:", python_startup)
self.shell.showtraceback()
for startup_dir in startup_dirs[::-1]:
startup_files += glob.glob(os.path.join(startup_dir, '*.py'))
startup_files += glob.glob(os.path.join(startup_dir, '*.ipy'))
if not startup_files:
return
self.log.debug("Running startup files from %s...", startup_dir)
try:
for fname in sorted(startup_files):
self._exec_file(fname)
except:
self.log.warning("Unknown error in handling startup files:")
self.shell.showtraceback()
def _run_exec_files(self):
"""Run files from IPythonApp.exec_files"""
if not self.exec_files:
return
self.log.debug("Running files in IPythonApp.exec_files...")
try:
for fname in self.exec_files:
self._exec_file(fname)
except:
self.log.warning("Unknown error in handling IPythonApp.exec_files:")
self.shell.showtraceback()
def _run_cmd_line_code(self):
"""Run code or file specified at the command-line"""
if self.code_to_run:
line = self.code_to_run
try:
self.log.info("Running code given at command line (c=): %s" %
line)
self.shell.run_cell(line, store_history=False)
except:
self.log.warning("Error in executing line in user namespace: %s" %
line)
self.shell.showtraceback()
if not self.interact:
self.exit(1)
# Like Python itself, ignore the second if the first of these is present
elif self.file_to_run:
fname = self.file_to_run
if os.path.isdir(fname):
fname = os.path.join(fname, "__main__.py")
if not os.path.exists(fname):
self.log.warning("File '%s' doesn't exist", fname)
if not self.interact:
self.exit(2)
try:
self._exec_file(fname, shell_futures=True)
except:
self.shell.showtraceback(tb_offset=4)
if not self.interact:
self.exit(1)
def _run_module(self):
"""Run module specified at the command-line."""
if self.module_to_run:
# Make sure that the module gets a proper sys.argv as if it were
# run using `python -m`.
save_argv = sys.argv
sys.argv = [sys.executable] + self.extra_args
try:
self.shell.safe_run_module(self.module_to_run,
self.shell.user_ns)
finally:
sys.argv = save_argv

@ -0,0 +1,137 @@
# encoding: utf-8
"""
Simple utility for splitting user input. This is used by both inputsplitter and
prefilter.
Authors:
* Brian Granger
* Fernando Perez
"""
#-----------------------------------------------------------------------------
# Copyright (C) 2008-2011 The IPython Development Team
#
# Distributed under the terms of the BSD License. The full license is in
# the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------
#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
import re
import sys
from IPython.utils import py3compat
from IPython.utils.encoding import get_stream_enc
#-----------------------------------------------------------------------------
# Main function
#-----------------------------------------------------------------------------
# RegExp for splitting line contents into pre-char//first word-method//rest.
# For clarity, each group in on one line.
# WARNING: update the regexp if the escapes in interactiveshell are changed, as
# they are hardwired in.
# Although it's not solely driven by the regex, note that:
# ,;/% only trigger if they are the first character on the line
# ! and !! trigger if they are first char(s) *or* follow an indent
# ? triggers as first or last char.
line_split = re.compile(r"""
^(\s*) # any leading space
([,;/%]|!!?|\?\??)? # escape character or characters
\s*(%{0,2}[\w\.\*]*) # function/method, possibly with leading %
# to correctly treat things like '?%magic'
(.*?$|$) # rest of line
""", re.VERBOSE)
def split_user_input(line, pattern=None):
"""Split user input into initial whitespace, escape character, function part
and the rest.
"""
# We need to ensure that the rest of this routine deals only with unicode
encoding = get_stream_enc(sys.stdin, 'utf-8')
line = py3compat.cast_unicode(line, encoding)
if pattern is None:
pattern = line_split
match = pattern.match(line)
if not match:
# print "match failed for line '%s'" % line
try:
ifun, the_rest = line.split(None,1)
except ValueError:
# print "split failed for line '%s'" % line
ifun, the_rest = line, u''
pre = re.match(r'^(\s*)(.*)',line).groups()[0]
esc = ""
else:
pre, esc, ifun, the_rest = match.groups()
#print 'line:<%s>' % line # dbg
#print 'pre <%s> ifun <%s> rest <%s>' % (pre,ifun.strip(),the_rest) # dbg
return pre, esc or '', ifun.strip(), the_rest.lstrip()
class LineInfo(object):
"""A single line of input and associated info.
Includes the following as properties:
line
The original, raw line
continue_prompt
Is this line a continuation in a sequence of multiline input?
pre
Any leading whitespace.
esc
The escape character(s) in pre or the empty string if there isn't one.
Note that '!!' and '??' are possible values for esc. Otherwise it will
always be a single character.
ifun
The 'function part', which is basically the maximal initial sequence
of valid python identifiers and the '.' character. This is what is
checked for alias and magic transformations, used for auto-calling,
etc. In contrast to Python identifiers, it may start with "%" and contain
"*".
the_rest
Everything else on the line.
"""
def __init__(self, line, continue_prompt=False):
self.line = line
self.continue_prompt = continue_prompt
self.pre, self.esc, self.ifun, self.the_rest = split_user_input(line)
self.pre_char = self.pre.strip()
if self.pre_char:
self.pre_whitespace = '' # No whitespace allowed before esc chars
else:
self.pre_whitespace = self.pre
def ofind(self, ip):
"""Do a full, attribute-walking lookup of the ifun in the various
namespaces for the given IPython InteractiveShell instance.
Return a dict with keys: {found, obj, ospace, ismagic}
Note: can cause state changes because of calling getattr, but should
only be run if autocall is on and if the line hasn't matched any
other, less dangerous handlers.
Does cache the results of the call, so can be called multiple times
without worrying about *further* damaging state.
"""
return ip._ofind(self.ifun)
def __str__(self):
return "LineInfo [%s|%s|%s|%s]" %(self.pre, self.esc, self.ifun, self.the_rest)

Binary file not shown.

After

Width:  |  Height:  |  Size: 331 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 71 B

@ -0,0 +1,14 @@
"""Module with bad __all__
To test https://github.com/ipython/ipython/issues/9678
"""
def evil():
pass
def puppies():
pass
__all__ = [evil, # Bad
'puppies', # Good
]

@ -0,0 +1,12 @@
# -*- coding: utf-8 -*-
"""
Useless IPython extension to test installing and loading extensions.
"""
some_vars = {'arq': 185}
def load_ipython_extension(ip):
# set up simplified quantity input
ip.push(some_vars)
def unload_ipython_extension(ip):
ip.drop_by_id(some_vars)

@ -0,0 +1,4 @@
# coding: iso-8859-5
# (Unlikely to be the default encoding for most testers.)
# ±¶ÿàáâãäåæçèéêëìíîï <- Cyrillic characters
u = '®âðÄ'

@ -0,0 +1,4 @@
# coding: iso-8859-5
# (Unlikely to be the default encoding for most testers.)
# БЖџрстуфхцчшщъыьэюя <- Cyrillic characters
'Ўт№Ф'

@ -0,0 +1,46 @@
"""Minimal script to reproduce our nasty reference counting bug.
The problem is related to https://github.com/ipython/ipython/issues/141
The original fix for that appeared to work, but John D. Hunter found a
matplotlib example which, when run twice in a row, would break. The problem
were references held by open figures to internals of Tkinter.
This code reproduces the problem that John saw, without matplotlib.
This script is meant to be called by other parts of the test suite that call it
via %run as if it were executed interactively by the user. As of 2011-05-29,
test_run.py calls it.
"""
#-----------------------------------------------------------------------------
# Module imports
#-----------------------------------------------------------------------------
from IPython import get_ipython
#-----------------------------------------------------------------------------
# Globals
#-----------------------------------------------------------------------------
# This needs to be here because nose and other test runners will import
# this module. Importing this module has potential side effects that we
# want to prevent.
if __name__ == '__main__':
ip = get_ipython()
if not '_refbug_cache' in ip.user_ns:
ip.user_ns['_refbug_cache'] = []
aglobal = 'Hello'
def f():
return aglobal
cache = ip.user_ns['_refbug_cache']
cache.append(f)
def call_f():
for func in cache:
print('lowercased:',func().lower())

@ -0,0 +1,33 @@
"""Error script. DO NOT EDIT FURTHER! It will break exception doctests!!!"""
import sys
def div0():
"foo"
x = 1
y = 0
x/y
def sysexit(stat, mode):
raise SystemExit(stat, f"Mode = {mode}")
def bar(mode):
"bar"
if mode=='div':
div0()
elif mode=='exit':
try:
stat = int(sys.argv[2])
except:
stat = 1
sysexit(stat, mode)
else:
raise ValueError('Unknown mode')
if __name__ == '__main__':
try:
mode = sys.argv[1]
except IndexError:
mode = 'div'
bar(mode)

@ -0,0 +1,34 @@
"""Simple script to be run *twice*, to check reference counting bugs.
See test_run for details."""
import sys
# We want to ensure that while objects remain available for immediate access,
# objects from *previous* runs of the same script get collected, to avoid
# accumulating massive amounts of old references.
class C(object):
def __init__(self,name):
self.name = name
self.p = print
self.flush_stdout = sys.stdout.flush
def __del__(self):
self.p('tclass.py: deleting object:',self.name)
self.flush_stdout()
try:
name = sys.argv[1]
except IndexError:
pass
else:
if name.startswith('C'):
c = C(name)
#print >> sys.stderr, "ARGV:", sys.argv # dbg
# This next print statement is NOT debugging, we're making the check on a
# completely separate process so we verify by capturing stdout:
print('ARGV 1-:', sys.argv[1:])
sys.stdout.flush()

@ -0,0 +1,66 @@
from IPython.utils.capture import capture_output
import pytest
def test_alias_lifecycle():
name = 'test_alias1'
cmd = 'echo "Hello"'
am = _ip.alias_manager
am.clear_aliases()
am.define_alias(name, cmd)
assert am.is_alias(name)
assert am.retrieve_alias(name) == cmd
assert (name, cmd) in am.aliases
# Test running the alias
orig_system = _ip.system
result = []
_ip.system = result.append
try:
_ip.run_cell('%{}'.format(name))
result = [c.strip() for c in result]
assert result == [cmd]
finally:
_ip.system = orig_system
# Test removing the alias
am.undefine_alias(name)
assert not am.is_alias(name)
with pytest.raises(ValueError):
am.retrieve_alias(name)
assert (name, cmd) not in am.aliases
def test_alias_args_error():
"""Error expanding with wrong number of arguments"""
_ip.alias_manager.define_alias('parts', 'echo first %s second %s')
# capture stderr:
with capture_output() as cap:
_ip.run_cell('parts 1')
assert cap.stderr.split(":")[0] == "UsageError"
def test_alias_args_commented():
"""Check that alias correctly ignores 'commented out' args"""
_ip.run_line_magic("alias", "commentarg echo this is %%s a commented out arg")
with capture_output() as cap:
_ip.run_cell("commentarg")
# strip() is for pytest compat; testing via iptest patch IPython shell
# in testing.globalipapp and replace the system call which messed up the
# \r\n
assert cap.stdout.strip() == 'this is %s a commented out arg'
def test_alias_args_commented_nargs():
"""Check that alias correctly counts args, excluding those commented out"""
am = _ip.alias_manager
alias_name = 'comargcount'
cmd = 'echo this is %%s a commented out arg and this is not %s'
am.define_alias(alias_name, cmd)
assert am.is_alias(alias_name)
thealias = am.get_alias(alias_name)
assert thealias.nargs == 1

@ -0,0 +1,70 @@
# coding: utf-8
"""Tests for IPython.core.application"""
import os
import tempfile
from tempfile import TemporaryDirectory
from traitlets import Unicode
from IPython.core.application import BaseIPythonApplication
from IPython.testing import decorators as dec
@dec.onlyif_unicode_paths
def test_unicode_cwd():
"""Check that IPython starts with non-ascii characters in the path."""
wd = tempfile.mkdtemp(suffix=u"")
old_wd = os.getcwd()
os.chdir(wd)
#raise Exception(repr(os.getcwd()))
try:
app = BaseIPythonApplication()
# The lines below are copied from Application.initialize()
app.init_profile_dir()
app.init_config_files()
app.load_config_file(suppress_errors=False)
finally:
os.chdir(old_wd)
@dec.onlyif_unicode_paths
def test_unicode_ipdir():
"""Check that IPython starts with non-ascii characters in the IP dir."""
ipdir = tempfile.mkdtemp(suffix=u"")
# Create the config file, so it tries to load it.
with open(os.path.join(ipdir, "ipython_config.py"), "w", encoding="utf-8") as f:
pass
old_ipdir1 = os.environ.pop("IPYTHONDIR", None)
old_ipdir2 = os.environ.pop("IPYTHON_DIR", None)
os.environ["IPYTHONDIR"] = ipdir
try:
app = BaseIPythonApplication()
# The lines below are copied from Application.initialize()
app.init_profile_dir()
app.init_config_files()
app.load_config_file(suppress_errors=False)
finally:
if old_ipdir1:
os.environ["IPYTHONDIR"] = old_ipdir1
if old_ipdir2:
os.environ["IPYTHONDIR"] = old_ipdir2
def test_cli_priority():
with TemporaryDirectory() as td:
class TestApp(BaseIPythonApplication):
test = Unicode().tag(config=True)
# Create the config file, so it tries to load it.
with open(os.path.join(td, "ipython_config.py"), "w", encoding="utf-8") as f:
f.write("c.TestApp.test = 'config file'")
app = TestApp()
app.initialize(["--profile-dir", td])
assert app.test == "config file"
app = TestApp()
app.initialize(["--profile-dir", td, "--TestApp.test=cli"])
assert app.test == "cli"

@ -0,0 +1,316 @@
"""
Test for async helpers.
Should only trigger on python 3.5+ or will have syntax errors.
"""
import platform
from itertools import chain, repeat
from textwrap import dedent, indent
from unittest import TestCase
from IPython.testing.decorators import skip_without
import sys
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from IPython import get_ipython
ip = get_ipython()
iprc = lambda x: ip.run_cell(dedent(x)).raise_error()
iprc_nr = lambda x: ip.run_cell(dedent(x))
from IPython.core.async_helpers import _should_be_async
class AsyncTest(TestCase):
def test_should_be_async(self):
self.assertFalse(_should_be_async("False"))
self.assertTrue(_should_be_async("await bar()"))
self.assertTrue(_should_be_async("x = await bar()"))
self.assertFalse(
_should_be_async(
dedent(
"""
async def awaitable():
pass
"""
)
)
)
def _get_top_level_cases(self):
# These are test cases that should be valid in a function
# but invalid outside of a function.
test_cases = []
test_cases.append(('basic', "{val}"))
# Note, in all conditional cases, I use True instead of
# False so that the peephole optimizer won't optimize away
# the return, so CPython will see this as a syntax error:
#
# while True:
# break
# return
#
# But not this:
#
# while False:
# return
#
# See https://bugs.python.org/issue1875
test_cases.append(('if', dedent("""
if True:
{val}
""")))
test_cases.append(('while', dedent("""
while True:
{val}
break
""")))
test_cases.append(('try', dedent("""
try:
{val}
except:
pass
""")))
test_cases.append(('except', dedent("""
try:
pass
except:
{val}
""")))
test_cases.append(('finally', dedent("""
try:
pass
except:
pass
finally:
{val}
""")))
test_cases.append(('for', dedent("""
for _ in range(4):
{val}
""")))
test_cases.append(('nested', dedent("""
if True:
while True:
{val}
break
""")))
test_cases.append(('deep-nested', dedent("""
if True:
while True:
break
for x in range(3):
if True:
while True:
for x in range(3):
{val}
""")))
return test_cases
def _get_ry_syntax_errors(self):
# This is a mix of tests that should be a syntax error if
# return or yield whether or not they are in a function
test_cases = []
test_cases.append(('class', dedent("""
class V:
{val}
""")))
test_cases.append(('nested-class', dedent("""
class V:
class C:
{val}
""")))
return test_cases
def test_top_level_return_error(self):
tl_err_test_cases = self._get_top_level_cases()
tl_err_test_cases.extend(self._get_ry_syntax_errors())
vals = ('return', 'yield', 'yield from (_ for _ in range(3))',
dedent('''
def f():
pass
return
'''),
)
for test_name, test_case in tl_err_test_cases:
# This example should work if 'pass' is used as the value
with self.subTest((test_name, 'pass')):
iprc(test_case.format(val='pass'))
# It should fail with all the values
for val in vals:
with self.subTest((test_name, val)):
msg = "Syntax error not raised for %s, %s" % (test_name, val)
with self.assertRaises(SyntaxError, msg=msg):
iprc(test_case.format(val=val))
def test_in_func_no_error(self):
# Test that the implementation of top-level return/yield
# detection isn't *too* aggressive, and works inside a function
func_contexts = []
func_contexts.append(('func', False, dedent("""
def f():""")))
func_contexts.append(('method', False, dedent("""
class MyClass:
def __init__(self):
""")))
func_contexts.append(('async-func', True, dedent("""
async def f():""")))
func_contexts.append(('async-method', True, dedent("""
class MyClass:
async def f(self):""")))
func_contexts.append(('closure', False, dedent("""
def f():
def g():
""")))
def nest_case(context, case):
# Detect indentation
lines = context.strip().splitlines()
prefix_len = 0
for c in lines[-1]:
if c != ' ':
break
prefix_len += 1
indented_case = indent(case, ' ' * (prefix_len + 4))
return context + '\n' + indented_case
# Gather and run the tests
# yield is allowed in async functions, starting in Python 3.6,
# and yield from is not allowed in any version
vals = ('return', 'yield', 'yield from (_ for _ in range(3))')
success_tests = zip(self._get_top_level_cases(), repeat(False))
failure_tests = zip(self._get_ry_syntax_errors(), repeat(True))
tests = chain(success_tests, failure_tests)
for context_name, async_func, context in func_contexts:
for (test_name, test_case), should_fail in tests:
nested_case = nest_case(context, test_case)
for val in vals:
test_id = (context_name, test_name, val)
cell = nested_case.format(val=val)
with self.subTest(test_id):
if should_fail:
msg = ("SyntaxError not raised for %s" %
str(test_id))
with self.assertRaises(SyntaxError, msg=msg):
iprc(cell)
print(cell)
else:
iprc(cell)
def test_nonlocal(self):
# fails if outer scope is not a function scope or if var not defined
with self.assertRaises(SyntaxError):
iprc("nonlocal x")
iprc("""
x = 1
def f():
nonlocal x
x = 10000
yield x
""")
iprc("""
def f():
def g():
nonlocal x
x = 10000
yield x
""")
# works if outer scope is a function scope and var exists
iprc("""
def f():
x = 20
def g():
nonlocal x
x = 10000
yield x
""")
def test_execute(self):
iprc("""
import asyncio
await asyncio.sleep(0.001)
"""
)
def test_autoawait(self):
iprc("%autoawait False")
iprc("%autoawait True")
iprc("""
from asyncio import sleep
await sleep(0.1)
"""
)
if sys.version_info < (3, 9) and platform.python_implementation() != "PyPy":
# new pgen parser in 3.9 does not raise MemoryError on too many nested
# parens anymore
def test_memory_error(self):
with self.assertRaises(MemoryError):
iprc("(" * 200 + ")" * 200)
@skip_without('curio')
def test_autoawait_curio(self):
iprc("%autoawait curio")
@skip_without('trio')
def test_autoawait_trio(self):
iprc("%autoawait trio")
@skip_without('trio')
def test_autoawait_trio_wrong_sleep(self):
iprc("%autoawait trio")
res = iprc_nr("""
import asyncio
await asyncio.sleep(0)
""")
with self.assertRaises(TypeError):
res.raise_error()
@skip_without('trio')
def test_autoawait_asyncio_wrong_sleep(self):
iprc("%autoawait asyncio")
res = iprc_nr("""
import trio
await trio.sleep(0)
""")
with self.assertRaises(RuntimeError):
res.raise_error()
def tearDown(self):
ip.loop_runner = "asyncio"

@ -0,0 +1,66 @@
"""These kinds of tests are less than ideal, but at least they run.
This was an old test that was being run interactively in the top-level tests/
directory, which we are removing. For now putting this here ensures at least
we do run the test, though ultimately this functionality should all be tested
with better-isolated tests that don't rely on the global instance in iptest.
"""
from IPython.core.splitinput import LineInfo
from IPython.core.prefilter import AutocallChecker
def doctest_autocall():
"""
In [1]: def f1(a,b,c):
...: return a+b+c
...:
In [2]: def f2(a):
...: return a + a
...:
In [3]: def r(x):
...: return True
...:
In [4]: ;f2 a b c
Out[4]: 'a b ca b c'
In [5]: assert _ == "a b ca b c"
In [6]: ,f1 a b c
Out[6]: 'abc'
In [7]: assert _ == 'abc'
In [8]: print(_)
abc
In [9]: /f1 1,2,3
Out[9]: 6
In [10]: assert _ == 6
In [11]: /f2 4
Out[11]: 8
In [12]: assert _ == 8
In [12]: del f1, f2
In [13]: ,r a
Out[13]: True
In [14]: assert _ == True
In [15]: r'a'
Out[15]: 'a'
In [16]: assert _ == 'a'
"""
def test_autocall_should_ignore_raw_strings():
line_info = LineInfo("r'a'")
pm = ip.prefilter_manager
ac = AutocallChecker(shell=pm.shell, prefilter_manager=pm, config=pm.config)
assert ac.check(line_info) is None

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save