LIVY-171. Client side Python Job API.

Closes apache#183

.gitignore

@@ -18,3 +18,8 @@ reports/
 metastore_db/
 derby.log
 dependency-reduced-pom.xml
+
+# For python setup.py, which pollutes the source dirs.
+python-api/dist
+.eggs
+*.egg-info

.travis.yml

@@ -19,16 +19,18 @@ cache:
 
 before_install:
   - sudo apt-get -y install python3-pip python-dev
+  - sudo apt-get -y remove python-setuptools
+  - pip install --user --upgrade pip setuptools
+  - pip3 install --user --upgrade pip setuptools
   - pip install --user codecov cloudpickle
   - pip3 install --user cloudpickle
 
 install:
-  - mvn install -DskipTests=true -DskipITs=true -Dmaven.javadoc.skip=true -B -V
+  - mvn install -Dskip -DskipTests -DskipITs -Dmaven.javadoc.skip=true -B -V
 
 before_script:
   - pip install --user requests pytest flaky flake8
   - pip3 install --user requests pytest flaky
-  - flake8 . --exclude=./target,./resources --ignore=E127,E128
 
 script:
   - mvn verify

README.rst

@@ -42,14 +42,10 @@ MacOS:
   * Python 2.6+
   * R 3.x
 
-Required python packages for:
-
-install:
-  * cloudpickle
-
-tests:
+Required python packages for building Livy:
   * cloudpickle
   * requests
+  * flake8
   * flaky
   * pytest
 

pom.xml

@@ -178,6 +178,7 @@
     <module>core</module>
     <module>coverage</module>
     <module>examples</module>
+    <module>python-api</module>
     <module>repl</module>
     <module>rsc</module>
     <module>scala-api</module>
@@ -575,14 +576,13 @@
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-antrun-plugin</artifactId>
-          <version>1.6</version>
+          <version>1.8</version>
         </plugin>
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-jar-plugin</artifactId>
           <version>2.3.2</version>
         </plugin>
-
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-deploy-plugin</artifactId>
@@ -779,7 +779,6 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-antrun-plugin</artifactId>
-        <version>1.8</version>
         <executions>
           <!-- Cleans up files that tests append to (because we have two test plugins). -->
           <execution>

python-api/.gitattributes

@@ -0,0 +1,2 @@
+livy/_version.py export-subst
+src/main/python/livy/_version.py export-subst

python-api/MANIFEST.in

@@ -0,0 +1,3 @@
+include versioneer.py
+include livy/_version.py
+include src/main/python/livy/_version.py

python-api/pom.xml

@@ -0,0 +1,138 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+     xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>com.cloudera.livy</groupId>
+    <artifactId>livy-main</artifactId>
+    <version>0.3.0-SNAPSHOT</version>
+    <relativePath>../pom.xml</relativePath>
+  </parent>
+
+  <groupId>com.cloudera.livy</groupId>
+  <artifactId>livy-python-api</artifactId>
+  <version>0.3.0-SNAPSHOT</version>
+  <packaging>pom</packaging>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>exec-maven-plugin</artifactId>
+        <version>1.2.1</version>
+        <executions>
+          <execution>
+            <id>python-api install</id>
+            <phase>compile</phase>
+            <goals>
+              <goal>exec</goal>
+            </goals>
+            <configuration>
+            <executable>python</executable>
+              <arguments>
+                <argument>setup.py</argument>
+                <argument>sdist</argument>
+              </arguments>
+            </configuration>
+          </execution>
+          <execution>
+            <id>python-api test</id>
+            <phase>test</phase>
+            <goals>
+              <goal>exec</goal>
+            </goals>
+            <configuration>
+              <executable>python</executable>
+              <skip>${skipTests}</skip>
+              <arguments>
+                <argument>setup.py</argument>
+                <argument>test</argument>
+              </arguments>
+            </configuration>
+          </execution>
+          <execution>
+            <id>python-api verify</id>
+            <phase>verify</phase>
+            <goals>
+              <goal>exec</goal>
+            </goals>
+            <configuration>
+              <executable>flake8</executable>
+              <skip>${skip}</skip>
+              <arguments>
+                <argument>--exclude=target,resources,.eggs</argument>
+                <argument>--ignore=E127,E128</argument>
+                <argument>.</argument>
+              </arguments>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-antrun-plugin</artifactId>
+        <executions>
+          <!-- Cleans up files that tests append to (because we have two test plugins). -->
+          <execution>
+            <id>python-api clean</id>
+            <phase>clean</phase>
+            <goals>
+              <goal>run</goal>
+            </goals>
+            <configuration>
+              <target>
+                <delete file="${project.basedir}/dist" quiet="true" />
+                <delete file="${project.basedir}/.eggs" quiet="true" />
+                <delete file="${project.basedir}/src/main/python/*.egg-info" quiet="true" />
+              </target>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <configuration>
+          <skipTests>true</skipTests>
+        </configuration>
+      </plugin>
+
+      <plugin>
+        <groupId>org.scalatest</groupId>
+        <artifactId>scalatest-maven-plugin</artifactId>
+        <configuration>
+          <skipTests>true</skipTests>
+        </configuration>
+      </plugin>
+
+      <plugin>
+        <groupId>org.jacoco</groupId>
+        <artifactId>jacoco-maven-plugin</artifactId>
+        <configuration>
+          <skip>true</skip>
+        </configuration>
+      </plugin>
+
+      <plugin>
+        <groupId>org.scalastyle</groupId>
+        <artifactId>scalastyle-maven-plugin</artifactId>
+        <configuration>
+          <skip>true</skip>
+        </configuration>
+      </plugin>
+
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-checkstyle-plugin</artifactId>
+        <configuration>
+          <skip>true</skip>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

python-api/setup.cfg

@@ -0,0 +1,6 @@
+[aliases]
+test=pytest
+
+[tool:pytest]
+addopts = --verbose
+python_files = src/test/python/*/*.py

python-api/setup.py

@@ -0,0 +1,58 @@
+#
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from setuptools import setup
+
+DESCRIPTION = "A simple Python API for Livy powered by requests"
+
+CLASSIFIERS = [
+    'Development Status :: 1 - Planning',
+    'Intended Audience :: Developers',
+    'Operating System :: OS Independent',
+    'Programming Language :: Python :: 2.7',
+    'Topic :: Software Development :: Libraries :: Python Modules',
+]
+
+requirements = [
+    'cloudpickle>=0.2.1',
+    'configparser>=3.5.0',
+    'future>=0.15.2',
+    'futures>=3.0.5',
+    'requests>=2.10.0',
+    'responses>=0.5.1',
+]
+
+setup(
+    name='livy-python-api',
+    version="0.3.0-SNAPSHOT",
+    packages=["livy", "livy-tests"],
+    package_dir={
+        "": "src/main/python",
+        "livy-tests": "src/test/python/livy-tests",
+    },
+    url='https://github.com/cloudera/livy',
+    author_email='[email protected]',
+    license='Apache License, Version 2.0',
+    description=DESCRIPTION,
+    platforms=['any'],
+    keywords='livy pyspark development',
+    classifiers=CLASSIFIERS,
+    install_requires=requirements,
+    setup_requires=['pytest-runner', 'flake8'],
+    tests_require=['pytest']
+)

python-api/src/main/python/livy/__init__.py

@@ -0,0 +1,17 @@
+#
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#