HBASE-29454 Update hbase-examples scripts to be compatible with Python 3

Author: kgeisz

.gitignore

@@ -24,3 +24,4 @@ linklint/
 tmp
 **/.flattened-pom.xml
 .vscode/
+**/__pycache__

dev-support/blanks-eol-ignore.txt

@@ -20,5 +20,5 @@
 .*/gen-cpp/.*
 .*/gen-perl/.*
 .*/gen-php/.*
-.*/gen-py/.*
+.*/gen_py/.*
 .*/gen-rb/.*

dev-support/blanks-tabs-ignore.txt

@@ -20,7 +20,7 @@
 .*/gen-cpp/.*
 .*/gen-perl/.*
 .*/gen-php/.*
-.*/gen-py/.*
+.*/gen_py/.*
 .*/gen-rb/.*
 # we have tabs in asciidoc, not sure whether it is OK to replace them with spaces
 src/main/asciidoc/.*

hbase-examples/README.txt

@@ -17,6 +17,12 @@ Example code.
         ${HBASE_ROOT}/hbase-thrift/src/main/resources/org/apache/hadoop/hbase/thrift/Hbase.thrift
     and re-placed at the corresponding paths. You should not have to do this generally.
 
+    For Python, modules with a dash cannot be imported. Instead, you must use an underscore or no
+    dash at all. You can generate the code in a custom directory with the following:
+    mkdir gen_py
+    thrift --gen py --out gen_py \
+        ${HBASE_ROOT}/hbase-thrift/src/main/resources/org/apache/hadoop/hbase/thrift/Hbase.thrift
+
     Before you run any Thrift examples, find a running HBase Thrift server (and a running
     hbase cluster for this server to talk to -- at a minimum start a standalone instance
     by doing ./bin/start-hbase.sh). If you start one locally (bin/hbase thrift start),
@@ -38,8 +44,8 @@ Example code.
       1. Modify the import path in the file to point to {$THRIFT_HOME}/lib/rb/lib.
       2. Execute {ruby DemoClient.rb} (or {ruby DemoClient.rb <host> <port>}).
 
-    * Python: hbase-examples/src/main/python/DemoClient.py
-      1. Modify the added system path in the file to point to {$THRIFT_HOME}/lib/py/build/lib.[YOUR SYSTEM]
+    * Python: hbase-examples/src/main/python/thrift1/DemoClient.py and hbase-examples/src/main/python/thrift2/DemoClient.py
+      1. Install the thrift package to your local Python environment: pip3 install thrift
       2. Execute {python DemoClient.py <host> <port>}.
 
     * PHP: hbase-examples/src/main/php/DemoClient.php

hbase-examples/src/main/python/thrift1/DemoClient.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 '''
   Licensed to the Apache Software Foundation (ASF) under one
   or more contributor license agreements.  See the NOTICE file
@@ -20,30 +20,22 @@
 import sys
 import time
 import os
-
-# Modify this import path to point to the correct location to thrift.
-thrift_path = os.path.abspath('/Users/sergey/Downloads/thrift/lib/py/build/lib.macosx-10.8-intel-2.7')
-sys.path.append(thrift_path)
-gen_py_path = os.path.abspath('gen-py')
-sys.path.append(gen_py_path)
-
-from thrift import Thrift
 from thrift.transport import TSocket, TTransport
 from thrift.protocol import TBinaryProtocol
-from hbase.ttypes import TThriftServerType
-from hbase.Hbase import Client, ColumnDescriptor, Mutation
+from gen_py.hbase import ttypes
+from gen_py.hbase.Hbase import Client, ColumnDescriptor, Mutation
 
 def printVersions(row, versions):
-  print "row: " + row + ", values: ",
+  print("row: " + row + ", values: ", end=' ')
   for cell in versions:
-    print cell.value + "; ",
-  print
+    print(cell.value + "; ", end=' ')
+  print()
 
 def printRow(entry):
-  print "row: " + entry.row + ", cols:",
+  print("row: " + entry.row + ", cols:", end=' ')
   for k in sorted(entry.columns):
-    print k + " => " + entry.columns[k].value,
-  print
+    print(k + " => " + entry.columns[k].value, end=' ')
+  print()
 
 
 def demo_client(host, port, is_framed_transport):
@@ -68,22 +60,22 @@ def demo_client(host, port, is_framed_transport):
 
   # Check Thrift Server Type
   serverType = client.getThriftServerType()
-  if serverType != TThriftServerType.ONE:
-    raise Exception("Mismatch between client and server, server type is %s" % serverType)
+  if serverType != ttypes.TThriftServerType.ONE:
+    raise RuntimeError(f"Mismatch between client and server, server type is {serverType}")
 
   t = "demo_table"
 
   #
   # Scan all tables, look for the demo table and delete it.
   #
-  print "scanning tables..."
+  print("scanning tables...")
   for table in client.getTableNames():
-    print "  found: %s" %(table)
+    print(f"  found: {table}")
     if table == t:
       if client.isTableEnabled(table):
-        print "    disabling table: %s"  %(t)
+        print(f"    disabling table: {t}")
         client.disableTable(table)
-      print "    deleting table: %s"  %(t)
+      print(f"    deleting table: {t}")
       client.deleteTable(table)
 
   columns = []
@@ -96,16 +88,16 @@ def demo_client(host, port, is_framed_transport):
   columns.append(col)
 
   try:
-    print "creating table: %s" %(t)
+    print(f"creating table: {t}")
     client.createTable(t, columns)
-  except AlreadyExists, ae:
-    print "WARN: " + ae.message
+  except ttypes.AlreadyExists as ae:
+    print("WARN: " + ae.message)
 
   cols = client.getColumnDescriptors(t)
-  print "column families in %s" %(t)
+  print(f"column families in {t}")
   for col_name in cols.keys():
     col = cols[col_name]
-    print "  column: %s, maxVer: %d" % (col.name, col.maxVersions)
+    print(f"  column: {col.name}, maxVer: {col.maxVersions}")
 
   dummy_attributes = {}
   #
@@ -116,7 +108,7 @@ def demo_client(host, port, is_framed_transport):
 
   # non-utf8 is fine for data
   mutations = [Mutation(column="entry:foo",value=invalid)]
-  print str(mutations)
+  print(str(mutations))
   client.mutateRow(t, "foo", mutations, dummy_attributes)
 
   # try empty strings
@@ -131,25 +123,25 @@ def demo_client(host, port, is_framed_transport):
   try:
     mutations = [Mutation(column="entry:foo", value=invalid)]
     client.mutateRow(t, invalid, mutations, dummy_attributes)
-  except ttypes.IOError, e:
-    print 'expected exception: %s' %(e.message)
+  except ttypes.IOError as e:
+    print(f'expected exception: {e.message}')
 
   # Run a scanner on the rows we just created
-  print "Starting scanner..."
+  print("Starting scanner...")
   scanner = client.scannerOpen(t, "", ["entry:"], dummy_attributes)
 
   r = client.scannerGet(scanner)
   while r:
     printRow(r[0])
     r = client.scannerGet(scanner)
-  print "Scanner finished"
+  print("Scanner finished")
 
   #
   # Run some operations on a bunch of rows.
   #
   for e in range(100, 0, -1):
     # format row keys as "00000" to "00100"
-    row = "%0.5d" % (e)
+    row = f"{row:05}"
 
     mutations = [Mutation(column="unused:", value="DELETE_ME")]
     client.mutateRow(t, row, mutations, dummy_attributes)
@@ -187,15 +179,15 @@ def demo_client(host, port, is_framed_transport):
     r = client.get(t, row, "entry:foo", dummy_attributes)
     # just to be explicit, we get lists back, if it's empty there was no matching row.
     if len(r) > 0:
-      raise "shouldn't get here!"
+      raise RuntimeError("shouldn't get here!")
 
   columnNames = []
   for (col, desc) in client.getColumnDescriptors(t).items():
-    print "column with name: "+desc.name
-    print desc
+    print("column with name: "+desc.name)
+    print(desc)
     columnNames.append(desc.name+":")
 
-  print "Starting scanner..."
+  print("Starting scanner...")
   scanner = client.scannerOpenWithStop(t, "00020", "00040", columnNames, dummy_attributes)
 
   r = client.scannerGet(scanner)
@@ -204,16 +196,14 @@ def demo_client(host, port, is_framed_transport):
     r = client.scannerGet(scanner)
 
   client.scannerClose(scanner)
-  print "Scanner finished"
+  print("Scanner finished")
 
   transport.close()
 
 
 if __name__ == '__main__':
-
-  import sys
   if len(sys.argv) < 3:
-    print 'usage: %s <host> <port>' % __file__
+    print(f'usage: {__file__} <host> <port>')
     sys.exit(1)
 
   host = sys.argv[1]

hbase-examples/src/main/python/thrift1/demo_hbase_thrift_over_http_tls.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+'''
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+'''
+#
+# test thrift server over HTTP with SSL in place
+#
+# presumes thrift python lib is installed
+# presumes you have access to hbase thrift binding (i.e. you add gen_py to PYTHONPATH)
+# presumes thrift proxy is running on port 9090
+# presumes thrift proxy is running over https
+# presumes access to create and use tables in a namespace 'test'
+#
+# usage:
+# ./demo_hbase_thrift_over_http_tls.py host-running-thrift1.example.com
+import sys
+
+from thrift.transport import THttpClient
+from thrift.protocol import TBinaryProtocol
+from gen_py.hbase import Hbase
+from gen_py.hbase.ttypes import ColumnDescriptor
+from gen_py.hbase.ttypes import Mutation
+
+print("[INFO] setup connection")
+transport = THttpClient.THttpClient(f"https://{sys.argv[1]}:9090")
+client = Hbase.Client(TBinaryProtocol.TBinaryProtocol(transport))
+
+table='test:thrift_proxy_demo'
+
+print("[INFO] start client")
+transport.open()
+
+print("[INFO] list the current tables")
+print(client.getTableNames())
+
+print("[INFO] create a table, place some data")
+client.createTable(table, [ColumnDescriptor(name ='family1:')])
+client.mutateRow(table, 'row1',
+                 [Mutation(column = 'family1:cq1', value = 'foo'),
+                  Mutation(column = 'family1:cq2', value = 'foo')], {})
+client.mutateRow(table, 'row2',
+                 [Mutation(column = 'family1:cq1', value = 'bar'),
+                  Mutation(column = 'family1:cq2', value = 'bar')], {})
+client.mutateRow(table, 'row3',
+                 [Mutation(column = 'family1:cq1', value = 'foo'),
+                  Mutation(column = 'family1:cq2', value = 'foo')], {})
+client.mutateRow(table, 'row4',
+                 [Mutation(column = 'family1:cq1', value = 'bar'),
+                  Mutation(column = 'family1:cq2', value = 'bar')], {})
+
+print("[INFO] scan")
+scan_id = client.scannerOpen(table, 'row1', [], {})
+for row in client.scannerGetList(scan_id, 25):
+  print(row)
+client.scannerClose(scan_id)
+print("[INFO] get")
+for row in client.getRow(table, 'row3', {}):
+  print(row)
+
+print("[INFO] clean up")
+client.disableTable(table)
+client.deleteTable(table)

hbase-examples/src/main/python/thrift1/gen-py/hbase/__init__.py

@@ -0,0 +1,18 @@
+'''
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+'''
+__all__ = ['ttypes', 'constants', 'Hbase']