Parse SQL comments in first line

NEWS.rst

@@ -164,3 +164,9 @@ Deleted Plugin import left behind in 0.2.2
 * Bogus pseudo-SQL command `PERSIST` removed, replaced with `--persist` arg
 * Turn off echo of connection information with `displaycon` in config
 * Consistent support for {} variables (thanks Lucas)
+
+0.4.1
+~~~~~
+
+* Fixed .rst file location in MANIFEST.in 
+* Parse SQL comments in first line

README.rst

@@ -339,6 +339,51 @@ are provided by `PGSpecial`_.  Example:
 
 .. _meta-commands: https://www.postgresql.org/docs/9.6/static/app-psql.html#APP-PSQL-META-COMMANDS
 
+
+Options
+-------
+
+``-l`` / ``--connections``
+    List all active connections
+
+``-x`` / ``--close <session-name>`` 
+    Close named connection 
+
+``-c`` / ``--creator <creator-function>``
+    Specify creator function for new connection
+
+``-s`` / ``--section <section-name>``
+    Section of dsn_file to be used for generating a connection string
+
+``-p`` / ``--persist``
+    Create a table name in the database from the named DataFrame
+
+``--append``
+    Like ``--persist``, but appends to the table if it already exists 
+
+``-a`` / ``--connection_arguments <"{connection arguments}">``
+    Specify dictionary of connection arguments to pass to SQL driver
+
+``-f`` / ``--file <path>``
+    Run SQL from file at this path
+
+Caution 
+-------
+
+Comments
+~~~~~~~~
+
+Because ipyton-sql accepts ``--``-delimited options like ``--persist``, but ``--`` 
+is also the syntax to denote a SQL comment, the parser needs to make some assumptions.
+
+- If you try to pass an unsupported argument, like ``--lutefisk``, it will 
+  be interpreted as a SQL comment and will not throw an unsupported argument 
+  exception.
+- If the SQL statement begins with a first-line comment that looks like one 
+  of the accepted arguments - like ``%sql --persist is great!`` - it will be 
+  parsed like an argument, not a comment.  Moving the comment to the second 
+  line or later will avoid this.
+
 Installing
 ----------
 

setup.py

@@ -1,45 +1,47 @@
-from io import open
-from setuptools import setup, find_packages
 import os
+from io import open
+
+from setuptools import find_packages, setup
 
 here = os.path.abspath(os.path.dirname(__file__))
-README = open(os.path.join(here, 'README.rst'), encoding='utf-8').read()
-NEWS = open(os.path.join(here, 'NEWS.rst'), encoding='utf-8').read()
+README = open(os.path.join(here, "README.rst"), encoding="utf-8").read()
+NEWS = open(os.path.join(here, "NEWS.rst"), encoding="utf-8").read()
 
 
-version = '0.4.0'
+version = "0.4.1"
 
 install_requires = [
-    'prettytable<1',
-    'ipython>=1.0',
-    'sqlalchemy>=0.6.7',
-    'sqlparse',
-    'six',
-    'ipython-genutils>=0.1.0',
+    "prettytable<1",
+    "ipython>=1.0",
+    "sqlalchemy>=0.6.7",
+    "sqlparse",
+    "six",
+    "ipython-genutils>=0.1.0",
 ]
 
 
-setup(name='ipython-sql',
+setup(
+    name="ipython-sql",
     version=version,
     description="RDBMS access via IPython",
-    long_description=README + '\n\n' + NEWS,
-    long_description_content_type='text/x-rst',
+    long_description=README + "\n\n" + NEWS,
+    long_description_content_type="text/x-rst",
     classifiers=[
-        'Development Status :: 3 - Alpha',
-        'Environment :: Console',
-        'License :: OSI Approved :: MIT License',
-        'Topic :: Database',
-        'Topic :: Database :: Front-Ends',
-        'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 2',
+        "Development Status :: 3 - Alpha",
+        "Environment :: Console",
+        "License :: OSI Approved :: MIT License",
+        "Topic :: Database",
+        "Topic :: Database :: Front-Ends",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 2",
     ],
-    keywords='database ipython postgresql mysql',
-    author='Catherine Devlin',
-    author_email='catherine.devlin@gmail.com',
-    url='https://pypi.python.org/pypi/ipython-sql',
-    license='MIT',
-    packages=find_packages('src'),
-    package_dir = {'': 'src'},
+    keywords="database ipython postgresql mysql",
+    author="Catherine Devlin",
+    author_email="catherine.devlin@gmail.com",
+    url="https://pypi.python.org/pypi/ipython-sql",
+    license="MIT",
+    packages=find_packages("src"),
+    package_dir={"": "src"},
     include_package_data=True,
     zip_safe=False,
     install_requires=install_requires,

src/sql/magic.py

@@ -2,10 +2,14 @@ import json
 import re
 from string import Formatter
 
-from IPython.core.magic import (Magics, cell_magic, line_magic, magics_class,
-                                needs_local_scope)
-from IPython.core.magic_arguments import (argument, magic_arguments,
-                                          parse_argstring)
+from IPython.core.magic import (
+    Magics,
+    cell_magic,
+    line_magic,
+    magics_class,
+    needs_local_scope,
+)
+from IPython.core.magic_arguments import argument, magic_arguments, parse_argstring
 from IPython.display import display_javascript
 from sqlalchemy.exc import OperationalError, ProgrammingError
 
@@ -148,9 +152,13 @@ class SqlMagic(Magics, Configurable):
         ]
         cell_params = {}
         for variable in cell_variables:
-            cell_params[variable] = local_ns[variable]
+            if variable in local_ns:
+                cell_params[variable] = local_ns[variable]
+            else:
+                raise NameError(variable)
         cell = cell.format(**cell_params)
 
+        line = sql.parse.without_sql_comment(parser=self.execute.parser, line=line)
         args = parse_argstring(self.execute, line)
         if args.connections:
             return sql.connection.Connection.connections
@@ -267,8 +275,11 @@ class SqlMagic(Magics, Configurable):
 
         # Get the DataFrame from the user namespace
         if not frame_name:
-            raise SyntaxError("Syntax: %sql PERSIST <name_of_data_frame>")
-        frame = eval(frame_name, user_ns)
+            raise SyntaxError("Syntax: %sql --persist <name_of_data_frame>")
+        try:
+            frame = eval(frame_name, user_ns)
+        except SyntaxError:
+            raise SyntaxError("Syntax: %sql --persist <name_of_data_frame>")
         if not isinstance(frame, DataFrame) and not isinstance(frame, Series):
             raise TypeError("%s is not a Pandas DataFrame or Series" % frame_name)
 

src/sql/parse.py

@@ -1,5 +1,7 @@
+import itertools
 import json
 import re
+import shlex
 from os.path import expandvars
 
 import six
@@ -54,20 +56,33 @@ def parse(cell, config):
     return result
 
 
-# def parse_sql_flags(sql):
-#     words = sql.split()
-#     flags = {
-#         'persist': False,
-#         'result_var': None
-#     }
-#     if not words:
-#         return (flags, "")
-#     num_words = len(words)
-#     trimmed_sql = sql
-#     if words[0].lower() == 'persist':
-#         flags['persist'] = True
-#         trimmed_sql =  " ".join(words[1:])
-#     elif num_words >= 2 and words[1] == '<<':
-#         flags['result_var'] = words[0]
-#         trimmed_sql = " ".join(words[2:])
-#     return (flags, trimmed_sql.strip())
+def _option_strings_from_parser(parser):
+    """Extracts the expected option strings (-a, --append, etc) from argparse parser 
+
+    Thanks Martijn Pieters
+    https://stackoverflow.com/questions/28881456/how-can-i-list-all-registered-arguments-from-an-argumentparser-instance
+
+    :param parser: [description]
+    :type parser: IPython.core.magic_arguments.MagicArgumentParser 
+    """
+    opts = [a.option_strings for a in parser._actions]
+    return list(itertools.chain.from_iterable(opts))
+
+
+def without_sql_comment(parser, line):
+    """Strips -- comment from a line 
+
+    The argparser unfortunately expects -- to precede an option, 
+    but in SQL that delineates a comment.  So this removes comments 
+    so a line can safely be fed to the argparser.
+
+    :param line: A line of SQL, possibly mixed with option strings 
+    :type line: str 
+    """
+
+    args = _option_strings_from_parser(parser)
+    result = itertools.takewhile(
+        lambda word: (not word.startswith("--")) or (word in args),
+        shlex.split(line, posix=False),
+    )
+    return " ".join(result)

src/tests/test_magic.py

@@ -10,9 +10,7 @@ from sql.magic import SqlMagic
 
 def runsql(ip_session, statements):
     if isinstance(statements, str):
-        statements = [
-            statements,
-        ]
+        statements = [statements]
     for statement in statements:
         result = ip_session.run_line_magic("sql", "sqlite:// %s" % statement)
     return result  # returns only last result

src/tests/test_parse.py

@@ -4,7 +4,7 @@ from pathlib import Path
 
 from six.moves import configparser
 
-from sql.parse import connection_from_dsn_section, parse
+from sql.parse import connection_from_dsn_section, parse, without_sql_comment
 
 try:
     from traitlets.config.configurable import Configurable
@@ -101,3 +101,80 @@ def test_connection_from_dsn_section():
     assert result == "postgres://goesto11:seentheelephant@my.remote.host:5432/pgmain"
     result = connection_from_dsn_section(section="DB_CONFIG_2", config=DummyConfig())
     assert result == "mysql://thefin:fishputsfishonthetable@127.0.0.1/dolfin"
+
+
+class Bunch:
+    def __init__(self, **kwds):
+        self.__dict__.update(kwds)
+
+
+class ParserStub:
+    opstrs = [
+        [],
+        ["-l", "--connections"],
+        ["-x", "--close"],
+        ["-c", "--creator"],
+        ["-s", "--section"],
+        ["-p", "--persist"],
+        ["--append"],
+        ["-a", "--connection_arguments"],
+        ["-f", "--file"],
+    ]
+    _actions = [Bunch(option_strings=o) for o in opstrs]
+
+
+parser_stub = ParserStub()
+
+
+def test_without_sql_comment_plain():
+
+    line = "SELECT * FROM author"
+    assert without_sql_comment(parser=parser_stub, line=line) == line
+
+
+def test_without_sql_comment_with_arg():
+
+    line = "--file moo.txt --persist SELECT * FROM author"
+    assert without_sql_comment(parser=parser_stub, line=line) == line
+
+
+def test_without_sql_comment_with_comment():
+
+    line = "SELECT * FROM author -- uff da"
+    expected = "SELECT * FROM author"
+    assert without_sql_comment(parser=parser_stub, line=line) == expected
+
+
+def test_without_sql_comment_with_arg_and_comment():
+
+    line = "--file moo.txt --persist SELECT * FROM author -- uff da"
+    expected = "--file moo.txt --persist SELECT * FROM author"
+    assert without_sql_comment(parser=parser_stub, line=line) == expected
+
+
+def test_without_sql_comment_unspaced_comment():
+
+    line = "SELECT * FROM author --uff da"
+    expected = "SELECT * FROM author"
+    assert without_sql_comment(parser=parser_stub, line=line) == expected
+
+
+def test_without_sql_comment_dashes_in_string():
+
+    line = "SELECT '--very --confusing' FROM author -- uff da"
+    expected = "SELECT '--very --confusing' FROM author"
+    assert without_sql_comment(parser=parser_stub, line=line) == expected
+
+
+def test_without_sql_comment_with_arg_and_leading_comment():
+
+    line = "--file moo.txt --persist --comment, not arg"
+    expected = "--file moo.txt --persist"
+    assert without_sql_comment(parser=parser_stub, line=line) == expected
+
+
+def test_without_sql_persist():
+
+    line = "--persist my_table --uff da"
+    expected = "--persist my_table"
+    assert without_sql_comment(parser=parser_stub, line=line) == expected