diff --git a/ai_services/document_understanding/python/__init__.py b/ai_services/document_understanding/python/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ai_services/document_understanding/python/document_classification/object_storage_document_classification_demo.py b/ai_services/document_understanding/python/document_classification/object_storage_document_classification_demo.py deleted file mode 100644 index 8446e014..00000000 --- a/ai_services/document_understanding/python/document_classification/object_storage_document_classification_demo.py +++ /dev/null @@ -1,85 +0,0 @@ -# coding: utf-8 -# Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved. -# This software is dual-licensed to you under the Universal Permissive License (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl or Apache License 2.0 as shown at http://www.apache.org/licenses/LICENSE-2.0. You may choose either license. - -########################################################################## -# object_storage_document_classification_demo.py -# -# Supports Python 3 -########################################################################## -# Info: -# Object Storage Document Classification Processor Job creation using OCI AI Document Understanding service. -# -########################################################################## -# Application Command line(no parameter needed) -# python object_storage_document_classification_demo.py -########################################################################## - -""" -This python script provides an example of how to use OCI Document Understanding Service document classification feature. - -The configuration file used by service clients will be sourced from the default location (~/.oci/config) and the -CONFIG_PROFILE profile will be used. - -The sample attempts to classify a document located in object storage. - -""" -import oci -import uuid -import base64 - -# Setup basic variables -# Auth Config -CONFIG_PROFILE = "DEFAULT" -config = oci.config.from_file('~/.oci/config', CONFIG_PROFILE) - -# Compartment where processor job will be created -COMPARTMENT_ID = "" # e.g. "ocid1.compartment.oc1..aaaaaaaae5j73axsja5fnahbn23ilop3ynjkcg77mcvgryddz4pkh2t5ppaq"; - -def create_processor_job_callback(times_called, response): - print("Waiting for processor lifecycle state to go into succeeded state:", response.data) - -# Generate Document classification Feature -document_classification_feature = oci.ai_document.models.DocumentClassificationFeature() - -# Setup input location where document being processed is stored. -object_location = oci.ai_document.models.ObjectLocation() -object_location.namespace_name = "" # e.g. "axhh9gizbq5x" -object_location.bucket_name = "" # e.g "demo_examples" -object_location.object_name = "" # e.g "document_classification_demo.pdf" - -# Setup the output location where processor job results will be created -output_location = oci.ai_document.models.OutputLocation() -output_location.namespace_name = "" # e.g. "axk2tfhlrens" -output_location.bucket_name = "" # e.g "output" -output_location.prefix = "" # e.g "demo" - -# Create a processor_job for document_classification_detection feature -create_processor_job_details_document_classification_detection = oci.ai_document.models.CreateProcessorJobDetails( - display_name=str(uuid.uuid4()), - compartment_id=COMPARTMENT_ID, - input_location=oci.ai_document.models.ObjectStorageLocations(object_locations=[object_location]), - output_location=output_location, - processor_config=oci.ai_document.models.GeneralProcessorConfig(features=[document_classification_feature])) - -print("Calling create_processor with create_processor_job_details_document_classification_detection:", create_processor_job_details_document_classification_detection) -create_processor_response = aiservicedocument_client.create_processor_job_and_wait_for_state( - create_processor_job_details=create_processor_job_details_document_classification_detection, - wait_for_states=[oci.ai_document.models.ProcessorJob.LIFECYCLE_STATE_SUCCEEDED], - waiter_kwargs={"wait_callback": create_processor_job_callback}) - -print("processor call succeeded with status: {} and request_id: {}.".format(create_processor_response.status, create_processor_response.request_id)) -processor_job: oci.ai_document.models.ProcessorJob = create_processor_response.data -print("create_processor_job_details_document_classification_detection response: ", create_processor_response.data) - -print("Getting result json from the output_location") -object_storage_client = oci.object_storage.ObjectStorageClient(config=config) -get_object_response = object_storage_client.get_object(namespace_name=output_location.namespace_name, - bucket_name=output_location.bucket_name, - object_name="{}/{}/{}_{}/results/{}.json".format( - output_location.prefix, processor_job.id, - object_location.namespace_name, - object_location.bucket_name, - object_location.object_name)) - -print(str(get_object_response.data.content.decode())) \ No newline at end of file diff --git a/ai_services/document_understanding/python/document_classification/inline_document_classification_demo.py b/ai_services/document_understanding/python/inline_document_classification_processor_job_demo.py similarity index 50% rename from ai_services/document_understanding/python/document_classification/inline_document_classification_demo.py rename to ai_services/document_understanding/python/inline_document_classification_processor_job_demo.py index 0c2b655c..8a2102fb 100644 --- a/ai_services/document_understanding/python/document_classification/inline_document_classification_demo.py +++ b/ai_services/document_understanding/python/inline_document_classification_processor_job_demo.py @@ -1,5 +1,5 @@ # coding: utf-8 -# Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved. # This software is dual-licensed to you under the Universal Permissive License (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl or Apache License 2.0 as shown at http://www.apache.org/licenses/LICENSE-2.0. You may choose either license. ########################################################################## @@ -16,27 +16,38 @@ ########################################################################## """ -This python script provides an example of how to use OCI Document Understanding Service document classification feature +This sample script creates an asynchronous processor job to perform document classification on document_classification_demo.pdf +(found in the resources folder), which is supplied inline as a base64 encoded string. Successful execution of this +sample will store the results in the configured Object Storage bucket, under the output_location variable. -The configuration file used by service clients will be sourced from the default location (~/.oci/config) and the -CONFIG_PROFILE profile will be used. - -The sample attempts to classify an inline sample document. +This script is designed to be executed on a user's computer that is configured with Python 3 or later. The user's computer +must also be configured with a config file as described in README.md to authenticate with the OCI tenancy used to execute +the SDK requests in this script. The authentication config file used by this script will be sourced from the default location (~/.oci/config) +and the default profile in the config file will be used. +The Document Understanding service region invoked by this example is represented by an endpoint url defined by the variable 'endpoint' """ + import oci import uuid import base64 +import os -# Setup basic variables +# CONFIGURE THE JOB ------------------------------------------------------------ # Auth Config -CONFIG_PROFILE = "DEFAULT" +# Initialize the service client with the default profile in the config file +config = oci.config.from_file() +# To use a specific profile in the config file, use these instructions instead: +""" +CONFIG_PROFILE = "doctest_profile" # specify the name of the profile in the config file config = oci.config.from_file('~/.oci/config', CONFIG_PROFILE) +""" -# Compartment where processor job will be created (required) -COMPARTMENT_ID = "" # e.g. "ocid1.compartment.oc1..aaaaaaaae5j73axsja5fnahbn23ilop3ynjkcg77mcvgryddz4pkh2t5ppaq"; +# Compartment where processor job will be created +# TODO - set the compartment ID from your tenancy +COMPARTMENT_ID = "ocid1.compartment.oc1..aaaaaaaag2hxw73h2kz6nkqfs4yvt6sdufhko7ufc367kfgxtcwrcwz5xsgq" # e.g. "ocid1.compartment.oc1..aaaaaaaar3nriwuw4nmqbhe7alu3ou76vgujnq3cvjwjpcv2iivfbadcqfna"; -#sample document +# Encode the sample document for submitting inline document_classification_sample_string = None with open("resources/document_classification_demo.pdf", "rb") as document_file: document_classification_sample_string = base64.b64encode(document_file.read()).decode('utf-8') @@ -44,39 +55,48 @@ def create_processor_job_callback(times_called, response): print("Waiting for processor lifecycle state to go into succeeded state:", response.data) -aiservicedocument_client = oci.ai_document.AIServiceDocumentClientCompositeOperations(oci.ai_document.AIServiceDocumentClient(config=config)) +# Initialize client service_endpoint is optional, if not provided then client will point to region specified in oci config +# TODO - set the Document Understanding endpoint for the desired region. See +endpoint = "https://document.aiservice.us-sanjose-1.oci.oraclecloud.com" #e.g. "https://document.aiservice.us-ashburn-1.oci.oraclecloud.com" +aiservicedocument_client = oci.ai_document.AIServiceDocumentClientCompositeOperations(oci.ai_document.AIServiceDocumentClient(config=config, service_endpoint=endpoint)) -# Generate Document classification Feature +# Specify the Document Classification feature document_classification_feature = oci.ai_document.models.DocumentClassificationFeature() # Setup the output location where processor job results will be created +# TODO - set .namespace_name, .bucket_name, and .prefix for your tenancy output_location = oci.ai_document.models.OutputLocation() -output_location.namespace_name = "" # e.g. "axk2tfhlrens" -output_location.bucket_name = "" # e.g "output" -output_location.prefix = "" # e.g "demo" +output_location.namespace_name = "axcqscmddjqn" # e.g. "axk2tfhlrens" +output_location.bucket_name = "postman" # e.g "output" +output_location.prefix = "results-python" #e.g. "python-sdk" -# Create a processor_job for document_classification_detection feature -create_processor_job_details_document_classification_detection = oci.ai_document.models.CreateProcessorJobDetails( +# Create a processor job for document classification +create_processor_job_details_document_classification_extraction = oci.ai_document.models.CreateProcessorJobDetails( display_name=str(uuid.uuid4()), compartment_id=COMPARTMENT_ID, input_location=oci.ai_document.models.InlineDocumentContent(data=document_classification_sample_string), output_location=output_location, processor_config=oci.ai_document.models.GeneralProcessorConfig(features=[document_classification_feature])) -print("Calling create_processor with create_processor_job_details_document_classification_detection:", create_processor_job_details_document_classification_detection) +# EXECUTE THE JOB ------------------------------------------------------------ +# Initiate the processor job status and wait for it to succeed +print("Calling create_processor with create_processor_job_details_document_classification_extraction:", create_processor_job_details_document_classification_extraction, "\n\rWaiting for processor job to finish...") create_processor_response = aiservicedocument_client.create_processor_job_and_wait_for_state( - create_processor_job_details=create_processor_job_details_document_classification_detection, + create_processor_job_details=create_processor_job_details_document_classification_extraction, wait_for_states=[oci.ai_document.models.ProcessorJob.LIFECYCLE_STATE_SUCCEEDED], waiter_kwargs={"wait_callback": create_processor_job_callback}) -print("processor call succeeded with status: {} and request_id: {}.".format(create_processor_response.status, create_processor_response.request_id)) +# Get the processor job status and display it +print("Processor call succeeded with status: {} and request_id: {}.".format(create_processor_response.status, create_processor_response.request_id)) processor_job: oci.ai_document.models.ProcessorJob = create_processor_response.data -print("create_processor_job_details_document_classification_detection response: ", create_processor_response.data) +print("create_processor_job_details_document_classification_extraction response: ", create_processor_response.data) +# Get the text extraction results from the object storage output location and display it print("Getting defaultObject.json from the output_location") object_storage_client = oci.object_storage.ObjectStorageClient(config=config) get_object_response = object_storage_client.get_object(namespace_name=output_location.namespace_name, bucket_name=output_location.bucket_name, object_name="{}/{}/_/results/defaultObject.json".format( output_location.prefix, processor_job.id)) -print(str(get_object_response.data.content.decode())) \ No newline at end of file +print(str(get_object_response.data.content.decode())) +print("You can also view the results in the specified output location under the job id: ", processor_job.id) diff --git a/ai_services/document_understanding/python/inline_key_value_extraction_processor_job_demo.py b/ai_services/document_understanding/python/inline_key_value_extraction_processor_job_demo.py new file mode 100644 index 00000000..07145d88 --- /dev/null +++ b/ai_services/document_understanding/python/inline_key_value_extraction_processor_job_demo.py @@ -0,0 +1,101 @@ +# coding: utf-8 +# Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved. +# This software is dual-licensed to you under the Universal Permissive License (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl or Apache License 2.0 as shown at http://www.apache.org/licenses/LICENSE-2.0. You may choose either license. + +########################################################################## +# inline_key_value_extraction_processor_job_demo.py +# +# Supports Python 3 +########################################################################## +# Info: +# Inline Key Value extraction Processor Job creation using OCI AI Document Understanding service. +# +########################################################################## +# Application Command line(no parameter needed) +# python inline_key_value_extraction_processor_job_demo.py +########################################################################## + +""" +This sample script creates an asynchronous processor job to perform key value extraction on receipt.jpg +(found in the resources folder), which is supplied inline as a base64 encoded string. Successful execution of this +sample will create results in object storage as configured under the output_location variable. + +This script is designed to be executed on a user's computer that is configured with Python 3 or later. The user's computer +must also be configured with a config file as described in README.md to authenticate with the OCI tenancy used to execute +the SDK requests in this script. The authentication config file used by this script will be sourced from the default location (~/.oci/config) +and the default profile in the config file will be used. + +The Document Understanding service region invoked by this example is represented by an endpoint url defined by the variable 'endpoint' +""" + +import oci +import uuid +import base64 + +# CONFIGURE THE JOB ------------------------------------------------------------ +# Auth Config +# Initialize the service client with the default profile in the config file +config = oci.config.from_file() +# To use a specific profile in the config file, use these instructions instead: +""" +CONFIG_PROFILE = "doctest_profile" # specify the name of the profile in the config file +config = oci.config.from_file('~/.oci/config', CONFIG_PROFILE) +""" + +# Specify the Compartment where the processor job will be created +# TODO - set the compartment ID from your tenancy +COMPARTMENT_ID = "ocid1.compartment.oc1..aaaaaaaag2hxw73h2kz6nkqfs4yvt6sdufhko7ufc367kfgxtcwrcwz5xsgq" # e.g. "ocid1.compartment.oc1..aaaaaaaar3nriwuw4nmqbhe7alu3ou76vgujnq3cvjwjpcv2iivfbadcqfna"; + +# Encode the sample document for submitting inline +key_value_extraction_sample_string = None +with open("resources/key_value_extraction_demo.pdf", "rb") as document_file: + key_value_extraction_sample_string = base64.b64encode(document_file.read()).decode('utf-8') + +def create_processor_job_callback(times_called, response): + print("Waiting for processor lifecycle state to go into succeeded state:", response.data) + +# Initializing the client service_endpoint is optional. If not provided then client will point to region specified in oci config +# TODO - specify the Document Understanding endpoint for the desired region. See https://docs.oracle.com/en-us/iaas/api/#/en/document-understanding/20221109/ +endpoint = "https://document.aiservice.us-sanjose-1.oci.oraclecloud.com" #e.g. "https://document.aiservice.us-ashburn-1.oci.oraclecloud.com" +aiservicedocument_client = oci.ai_document.AIServiceDocumentClientCompositeOperations(oci.ai_document.AIServiceDocumentClient(config=config, service_endpoint=endpoint)) + +# Specify the Key-Value Extraction feature for this processor job +key_value_extraction_feature = oci.ai_document.models.DocumentKeyValueExtractionFeature() + +# Setup the output location where the processor job results will be created +# TODO - set .namespace_name, .bucket_name, and .prefix for your tenancy +output_location = oci.ai_document.models.OutputLocation() +output_location.namespace_name = "axcqscmddjqn" # e.g. "axk2tfhlrens" +output_location.bucket_name = "postman" # e.g "output" +output_location.prefix = "results-python" #e.g. "python-sdk" + +# Create a processor_job for key_value_extraction feature +create_processor_job_details_key_value_extraction = oci.ai_document.models.CreateProcessorJobDetails( + display_name=str(uuid.uuid4()), + compartment_id=COMPARTMENT_ID, + input_location=oci.ai_document.models.InlineDocumentContent(data=key_value_extraction_sample_string), + output_location=output_location, + processor_config=oci.ai_document.models.GeneralProcessorConfig(features=[key_value_extraction_feature])) + +# EXECUTE THE JOB ------------------------------------------------------------ +# Initiate the processor job status and wait for it to succeed +print("Calling create_processor with create_processor_job_details_key_value_extraction:", create_processor_job_details_key_value_extraction, "\n\rWaiting for processor job to finish...") +create_processor_response = aiservicedocument_client.create_processor_job_and_wait_for_state( + create_processor_job_details=create_processor_job_details_key_value_extraction, + wait_for_states=[oci.ai_document.models.ProcessorJob.LIFECYCLE_STATE_SUCCEEDED], + waiter_kwargs={"wait_callback": create_processor_job_callback}) + +# Get the processor job status and display the job response +print("Processor job succeeded with status: {} and request_id: {}.".format(create_processor_response.status, create_processor_response.request_id)) +processor_job: oci.ai_document.models.ProcessorJob = create_processor_response.data +print("This is the processor job response: ", create_processor_response.data) + +# Get the key-value extraction results from the object storage output location and display it +print("This is the output json retrieved from the output location:") +object_storage_client = oci.object_storage.ObjectStorageClient(config=config) +get_object_response = object_storage_client.get_object(namespace_name=output_location.namespace_name, + bucket_name=output_location.bucket_name, + object_name="{}/{}/_/results/defaultObject.json".format( + output_location.prefix, processor_job.id)) +print(str(get_object_response.data.content.decode())) +print("You can also view the results in the specified output location under the job id: ", processor_job.id) \ No newline at end of file diff --git a/ai_services/document_understanding/python/table_extraction/inline_table_extraction_demo.py b/ai_services/document_understanding/python/inline_table_extraction_processor_job_demo.py similarity index 50% rename from ai_services/document_understanding/python/table_extraction/inline_table_extraction_demo.py rename to ai_services/document_understanding/python/inline_table_extraction_processor_job_demo.py index 1f42312c..da48f48f 100644 --- a/ai_services/document_understanding/python/table_extraction/inline_table_extraction_demo.py +++ b/ai_services/document_understanding/python/inline_table_extraction_processor_job_demo.py @@ -1,61 +1,75 @@ # coding: utf-8 -# Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved. # This software is dual-licensed to you under the Universal Permissive License (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl or Apache License 2.0 as shown at http://www.apache.org/licenses/LICENSE-2.0. You may choose either license. ########################################################################## -# inline_table_extraction_demo.py +# inline_table_extraction_processor_job_demo.py # # Supports Python 3 ########################################################################## # Info: -# Inline table extraction Processor Job creation using OCI AI Document Understanding service. +# Inline Table extraction Processor Job creation using OCI AI Document Understanding service. # ########################################################################## # Application Command line(no parameter needed) -# python inline_table_extraction_demo.py +# python inline_table_extraction_processor_job_demo.py ########################################################################## """ -This python script provides an example of how to use OCI Document Understanding Service table extraction feature +This sample script creates an asynchronous processor job to perform table extraction on table_extraction_demo.pdf +(found in the resources folder), which is supplied inline as a base64 encoded string. Successful execution of this +sample will create results in object storage as configured under the output_location variable. -The configuration file used by service clients will be sourced from the default location (~/.oci/config) and the -CONFIG_PROFILE profile will be used. +This script is designed to be executed on a user's computer that is configured with Python 3 or later. The user's computer +must also be configured with a config file as described in README.md to authenticate with the OCI tenancy used to execute +the SDK requests in this script. The authentication config file used by this script will be sourced from the default location (~/.oci/config) +and the default profile in the config file will be used. -The sample attempts to extract table data from an inline document. -Successful run of this sample will create job results under object storage configured under output_location variable +The Document Understanding service region invoked by this example is represented by an endpoint url defined by the variable 'endpoint' """ + import oci import uuid import base64 -# Setup basic variables +# CONFIGURE THE JOB ------------------------------------------------------------ # Auth Config -CONFIG_PROFILE = "DEFAULT" +# Initialize the service client with the default profile in the config file +config = oci.config.from_file() +# To use a specific profile in the config file, use these instructions instead: +""" +CONFIG_PROFILE = "doctest_profile" # specify the name of the profile in the config file config = oci.config.from_file('~/.oci/config', CONFIG_PROFILE) +""" -# Compartment where processor job will be created (required) -COMPARTMENT_ID = "" # e.g. "ocid1.compartment.oc1..aaaaaaaae5j73axsja5fnahbn23ilop3ynjkcg77mcvgryddz4pkh2t5ppaq"; +# Compartment where processor job will be created +# TODO - set the compartment ID from your tenancy +COMPARTMENT_ID = "ocid1.compartment.oc1..aaaaaaaag2hxw73h2kz6nkqfs4yvt6sdufhko7ufc367kfgxtcwrcwz5xsgq" # e.g. "ocid1.compartment.oc1..aaaaaaaar3nriwuw4nmqbhe7alu3ou76vgujnq3cvjwjpcv2iivfbadcqfna"; -#sample document +# Encode the sample document for submitting inline table_extraction_sample_string = None -with open("resources/table_demo.jpg", "rb") as document_file: +with open("resources/table_extraction_demo.pdf", "rb") as document_file: table_extraction_sample_string = base64.b64encode(document_file.read()).decode('utf-8') def create_processor_job_callback(times_called, response): print("Waiting for processor lifecycle state to go into succeeded state:", response.data) -aiservicedocument_client = oci.ai_document.AIServiceDocumentClientCompositeOperations(oci.ai_document.AIServiceDocumentClient(config=config)) +# Initializing the client service_endpoint is optional. If not provided then client will point to region specified in oci config +# TODO - specify the Document Understanding endpoint for the desired region. See https://docs.oracle.com/en-us/iaas/api/#/en/document-understanding/20221109/ +endpoint = "https://document.aiservice.us-sanjose-1.oci.oraclecloud.com" #e.g. "https://document.aiservice.us-ashburn-1.oci.oraclecloud.com" +aiservicedocument_client = oci.ai_document.AIServiceDocumentClientCompositeOperations(oci.ai_document.AIServiceDocumentClient(config=config, service_endpoint=endpoint)) -# Table extraction Feature +# Specify the Table Extraction feature for this processor job table_extraction_feature = oci.ai_document.models.DocumentTableExtractionFeature() -# Setup the output location where processor job results will be created +# Setup the output location where the processor job results will be created +# TODO - set .namespace_name, .bucket_name, and .prefix for your tenancy output_location = oci.ai_document.models.OutputLocation() -output_location.namespace_name = "" # e.g. "axk2tfhlrens" -output_location.bucket_name = "" # e.g "output" -output_location.prefix = "" # e.g "demo" +output_location.namespace_name = "axcqscmddjqn" # e.g. "axk2tfhlrens" +output_location.bucket_name = "postman" # e.g "output" +output_location.prefix = "results-python" #e.g. "python-sdk" -# Create a processor_job for table_extraction feature +# Create a processor job for table extraction create_processor_job_details_table_extraction = oci.ai_document.models.CreateProcessorJobDetails( display_name=str(uuid.uuid4()), compartment_id=COMPARTMENT_ID, @@ -63,20 +77,25 @@ def create_processor_job_callback(times_called, response): output_location=output_location, processor_config=oci.ai_document.models.GeneralProcessorConfig(features=[table_extraction_feature])) -print("Calling create_processor with create_processor_job_details_table_extraction:", create_processor_job_details_table_extraction) +# EXECUTE THE JOB ------------------------------------------------------------ +# Initiate the processor job status and wait for it to succeed +print("Calling create_processor with create_processor_job_details_table_extraction:", create_processor_job_details_table_extraction, "\n\rWaiting for processor job to finish...") create_processor_response = aiservicedocument_client.create_processor_job_and_wait_for_state( create_processor_job_details=create_processor_job_details_table_extraction, wait_for_states=[oci.ai_document.models.ProcessorJob.LIFECYCLE_STATE_SUCCEEDED], waiter_kwargs={"wait_callback": create_processor_job_callback}) +# Get the processor job status and display the job response print("processor call succeeded with status: {} and request_id: {}.".format(create_processor_response.status, create_processor_response.request_id)) processor_job: oci.ai_document.models.ProcessorJob = create_processor_response.data print("create_processor_job_details_table_extraction response: ", create_processor_response.data) +# Get the table extraction results from the object storage output location and display it print("Getting defaultObject.json from the output_location") object_storage_client = oci.object_storage.ObjectStorageClient(config=config) get_object_response = object_storage_client.get_object(namespace_name=output_location.namespace_name, bucket_name=output_location.bucket_name, object_name="{}/{}/_/results/defaultObject.json".format( output_location.prefix, processor_job.id)) -print(str(get_object_response.data.content.decode())) \ No newline at end of file +print(str(get_object_response.data.content.decode())) +print("You can also view the results in the specified output location under the job id: ", processor_job.id) \ No newline at end of file diff --git a/ai_services/document_understanding/python/inline_text_extraction_processor_job_demo.py b/ai_services/document_understanding/python/inline_text_extraction_processor_job_demo.py new file mode 100644 index 00000000..a764e73c --- /dev/null +++ b/ai_services/document_understanding/python/inline_text_extraction_processor_job_demo.py @@ -0,0 +1,103 @@ +# coding: utf-8 +# Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved. +# This software is dual-licensed to you under the Universal Permissive License (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl or Apache License 2.0 as shown at http://www.apache.org/licenses/LICENSE-2.0. You may choose either license. + +########################################################################## +# inline_text_extraction_processor_job_demo.py +# +# Supports Python 3 +########################################################################## +# Info: +# Inline Text extraction Processor Job creation using OCI AI Document Understanding service. +# +########################################################################## +# Application Command line(no parameter needed) +# python inline_text_extraction_processor_job_demo.py +########################################################################## + +""" +This sample script creates an asynchronous processor job to perform text extraction for inline cover.pdf +(found in the resources folder), which is supplied inline as a base64 encoded string. Successful execution of this +sample will create results in object storage as configured under the output_location variable. + +This script is designed to be executed on a user's computer that is configured with Python 3 or later. The user's computer +must also be configured with a config file as described in README.md to authenticate with the OCI tenancy used to execute +the SDK requests in this script. The authentication config file used by this script will be sourced from the default location (~/.oci/config) +and the default profile in the config file will be used. + +The Document Understanding service region invoked by this example is represented by an endpoint url defined by the variable 'endpoint' +""" + +import oci +import uuid +import base64 + +# CONFIGURE THE JOB ------------------------------------------------------------ +# Auth Config +# Initialize the service client with the default profile in the config file +config = oci.config.from_file() +# To use a specific profile in the config file, use these instructions instead: +""" +CONFIG_PROFILE = "doctest_profile" # specify the name of the profile in the config file +config = oci.config.from_file('~/.oci/config', CONFIG_PROFILE) +""" + +# Compartment where processor job will be created +# TODO - set the compartment ID from your tenancy +COMPARTMENT_ID = "ocid1.compartment.oc1..aaaaaaaag2hxw73h2kz6nkqfs4yvt6sdufhko7ufc367kfgxtcwrcwz5xsgq" # e.g. "ocid1.compartment.oc1..aaaaaaaar3nriwuw4nmqbhe7alu3ou76vgujnq3cvjwjpcv2iivfbadcqfna"; + +# Encode the sample document for submitting inline +text_extraction_sample_string = None +with open("resources/text_extraction_demo.jpg", "rb") as document_file: + text_extraction_sample_string = base64.b64encode(document_file.read()).decode('utf-8') + +def create_processor_job_callback(times_called, response): + print("Waiting for processor lifecycle state to go into succeeded state:", response.data) + +# Initialize client service_endpoint is optional, if not provided then client will point to region specified in oci config +# TODO - set the Document Understanding endpoint for the desired region. See +endpoint = "https://document.aiservice.us-sanjose-1.oci.oraclecloud.com" +aiservicedocument_client = oci.ai_document.AIServiceDocumentClientCompositeOperations(oci.ai_document.AIServiceDocumentClient(config=config, service_endpoint=endpoint)) + +# Specify the text extraction feature for this processor job +text_extraction_feature_searchable_pdf = oci.ai_document.models.DocumentTextExtractionFeature() +# Additionally, enable the Searchable PDF feature of text extraction to generate a searchable PDF file from the input document +text_extraction_feature_searchable_pdf.generate_searchable_pdf = True + +# Setup the output location where the processor job results will be created +# TODO - set .namespace_name, .bucket_name, and .prefix for your tenancy +output_location = oci.ai_document.models.OutputLocation() +output_location.namespace_name = "axcqscmddjqn" # e.g. "axk2tfhlrens" +output_location.bucket_name = "postman" # e.g "output" +output_location.prefix = "results-python" + +# Create a processor_job for text_extraction feature +create_processor_job_details_text_extraction = oci.ai_document.models.CreateProcessorJobDetails( + display_name=str(uuid.uuid4()), + compartment_id=COMPARTMENT_ID, + input_location=oci.ai_document.models.InlineDocumentContent(data=text_extraction_sample_string), + output_location=output_location, + processor_config=oci.ai_document.models.GeneralProcessorConfig(features=[text_extraction_feature_searchable_pdf])) + +# EXECUTE THE JOB ------------------------------------------------------------ +# Initiate the processor job status and wait for it to succeed +print("Calling create_processor with create_processor_job_details_text_extraction:", create_processor_job_details_text_extraction, "\n\rWaiting for processor job to finish...") +create_processor_response = aiservicedocument_client.create_processor_job_and_wait_for_state( + create_processor_job_details=create_processor_job_details_text_extraction, + wait_for_states=[oci.ai_document.models.ProcessorJob.LIFECYCLE_STATE_SUCCEEDED], + waiter_kwargs={"wait_callback": create_processor_job_callback}) + +# Get the processor job status and display the job response +print("processor call succeeded with status: {} and request_id: {}.".format(create_processor_response.status, create_processor_response.request_id)) +processor_job: oci.ai_document.models.ProcessorJob = create_processor_response.data +print("create_processor_job_details_text_extraction response: ", create_processor_response.data) + +# Get the table extraction results from the object storage output location and display it +print("Getting defaultObject.json from the output_location") +object_storage_client = oci.object_storage.ObjectStorageClient(config=config) +get_object_response = object_storage_client.get_object(namespace_name=output_location.namespace_name, + bucket_name=output_location.bucket_name, + object_name="{}/{}/_/results/defaultObject.json".format( + output_location.prefix, processor_job.id)) +print(str(get_object_response.data.content.decode())) +print("You can also view the results in the specified output location under the job id: ", processor_job.id) \ No newline at end of file diff --git a/ai_services/document_understanding/python/key_value_extraction/inline_key_value_extraction_demo.py b/ai_services/document_understanding/python/key_value_extraction/inline_key_value_extraction_demo.py deleted file mode 100644 index ff032927..00000000 --- a/ai_services/document_understanding/python/key_value_extraction/inline_key_value_extraction_demo.py +++ /dev/null @@ -1,84 +0,0 @@ -# coding: utf-8 -# Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved. -# This software is dual-licensed to you under the Universal Permissive License (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl or Apache License 2.0 as shown at http://www.apache.org/licenses/LICENSE-2.0. You may choose either license. - -########################################################################## -# inline_key_value_extraction_demo.py -# -# Supports Python 3 -########################################################################## -# Info: -# Inline Key Value extraction Processor Job creation using OCI AI Document Understanding service. -# -########################################################################## -# Application Command line(no parameter needed) -# python inline_key_value_extraction_demo.py -########################################################################## - -""" -This python script provides an example of how to use OCI Document Understanding Service key value extraction feature. - -The configuration file used by service clients will be sourced from the default location (~/.oci/config) and the -CONFIG_PROFILE profile will be used. - -The sample attempts extract key fields from an inline invoice document. -Successful run of this sample will create job results under object storage configured under output_location variable -""" -import oci -import uuid -import base64 - -# Setup basic variables -# Auth Config -CONFIG_PROFILE = "DEFAULT" -config = oci.config.from_file('~/.oci/config', CONFIG_PROFILE) - -# Compartment where processor job will be created (required) -COMPARTMENT_ID = "" # e.g. "ocid1.compartment.oc1..aaaaaaaae5j73axsja5fnahbn23ilop3ynjkcg77mcvgryddz4pkh2t5ppaq"; - -#sample document -key_value_extraction_sample_string = None -with open("resources/key_value_extraction_demo.pdf", "rb") as document_file: - key_value_extraction_sample_string = base64.b64encode(document_file.read()).decode('utf-8') - -def create_processor_job_callback(times_called, response): - print("Waiting for processor lifecycle state to go into succeeded state:", response.data) - -aiservicedocument_client = oci.ai_document.AIServiceDocumentClientCompositeOperations(oci.ai_document.AIServiceDocumentClient(config=config)) - -# Document Key-Value extraction Feature -key_value_extraction_feature = oci.ai_document.models.DocumentKeyValueExtractionFeature() - -# Setup the output location where processor job results will be created -output_location = oci.ai_document.models.OutputLocation() -output_location.namespace_name = "" # e.g. "axk2tfhlrens" -output_location.bucket_name = "" # e.g "output" -output_location.prefix = "" # e.g "demo" - -# Create a processor_job for invoice key_value_extraction feature. -# Note: If you want to use another key value extraction feature, set document_type to "RECEIPT" "PASSPORT" or "DRIVER_ID". If you have a mix of document types, you can remove document_type -create_processor_job_details_key_value_extraction = oci.ai_document.models.CreateProcessorJobDetails( - display_name=str(uuid.uuid4()), - compartment_id=COMPARTMENT_ID, - input_location=oci.ai_document.models.InlineDocumentContent(data=key_value_extraction_sample_string), - output_location=output_location, - processor_config=oci.ai_document.models.GeneralProcessorConfig(features=[key_value_extraction_feature], - document_type="INVOICE")) - -print("Calling create_processor with create_processor_job_details_key_value_extraction:", create_processor_job_details_key_value_extraction) -create_processor_response = aiservicedocument_client.create_processor_job_and_wait_for_state( - create_processor_job_details=create_processor_job_details_key_value_extraction, - wait_for_states=[oci.ai_document.models.ProcessorJob.LIFECYCLE_STATE_SUCCEEDED], - waiter_kwargs={"wait_callback": create_processor_job_callback}) - -print("processor call succeeded with status: {} and request_id: {}.".format(create_processor_response.status, create_processor_response.request_id)) -processor_job: oci.ai_document.models.ProcessorJob = create_processor_response.data -print("create_processor_job_details_key_value_extraction response: ", create_processor_response.data) - -print("Getting defaultObject.json from the output_location") -object_storage_client = oci.object_storage.ObjectStorageClient(config=config) -get_object_response = object_storage_client.get_object(namespace_name=output_location.namespace_name, - bucket_name=output_location.bucket_name, - object_name="{}/{}/_/results/defaultObject.json".format( - output_location.prefix, processor_job.id)) -print(str(get_object_response.data.content.decode())) \ No newline at end of file diff --git a/ai_services/document_understanding/python/key_value_extraction/object_storage_key_value_extraction_demo.py b/ai_services/document_understanding/python/key_value_extraction/object_storage_key_value_extraction_demo.py deleted file mode 100644 index bb3d459f..00000000 --- a/ai_services/document_understanding/python/key_value_extraction/object_storage_key_value_extraction_demo.py +++ /dev/null @@ -1,88 +0,0 @@ -# coding: utf-8 -# Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved. -# This software is dual-licensed to you under the Universal Permissive License (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl or Apache License 2.0 as shown at http://www.apache.org/licenses/LICENSE-2.0. You may choose either license. - -########################################################################## -# object_storage_key_value_extraction_demo.py -# -# Supports Python 3 -########################################################################## -# Info: -# Object Storage Key Value extraction Processor Job creation using OCI AI Document Understanding service. -# -########################################################################## -# Application Command line(no parameter needed) -# python object_storage_key_value_extraction_demo.py -########################################################################## - -""" -This python script provides an example of how to use OCI Document Understanding Service key value extraction feature. - -The configuration file used by service clients will be sourced from the default location (~/.oci/config) and the -CONFIG_PROFILE profile will be used. - -The sample attempts to extract key fields from an invoice document in object storage. -Successful run of this sample will create job results under object storage configured under output_location variable -""" -import oci -import uuid -import base64 - -# Setup basic variables -# Auth Config -CONFIG_PROFILE = "DEFAULT" -config = oci.config.from_file('~/.oci/config', CONFIG_PROFILE) - -# Compartment where processor job will be created (required) -COMPARTMENT_ID = "" # e.g. "ocid1.compartment.oc1..aaaaaaaae5j73axsja5fnahbn23ilop3ynjkcg77mcvgryddz4pkh2t5ppaq"; - -def create_processor_job_callback(times_called, response): - print("Waiting for processor lifecycle state to go into succeeded state:", response.data) - -# Setup input location where document being processed is stored. -object_location = oci.ai_document.models.ObjectLocation() -object_location.namespace_name = "" # e.g. "axhh9gizbq5x" -object_location.bucket_name = "" # e.g "demo_examples" -object_location.object_name = "" # e.g "key_value_extraction_demo.jpg - -aiservicedocument_client = oci.ai_document.AIServiceDocumentClientCompositeOperations(oci.ai_document.AIServiceDocumentClient(config=config)) - -# Document Key-Value extraction Feature -key_value_extraction_feature = oci.ai_document.models.DocumentKeyValueExtractionFeature() - -# Setup the output location where processor job results will be created -output_location = oci.ai_document.models.OutputLocation() -output_location.namespace_name = "" # e.g. "axk2tfhlrens" -output_location.bucket_name = "" # e.g "output" -output_location.prefix = "" # e.g "demo" - -# Create a processor_job for invoice key_value_extraction feature. -# Note: If you want to use another key value extraction feature, set document_type to "RECIEPT" "PASSPORT" or "DRIVER_ID". If you have a mix of document types, you can remove document_type -create_processor_job_details_key_value_extraction = oci.ai_document.models.CreateProcessorJobDetails( - display_name=str(uuid.uuid4()), - compartment_id=COMPARTMENT_ID, - input_location=oci.ai_document.models.ObjectStorageLocations(object_locations=[object_location]), - output_location=output_location, - processor_config=oci.ai_document.models.GeneralProcessorConfig(features=[key_value_extraction_feature], - document_type="INVOICE")) - -print("Calling create_processor with create_processor_job_details_key_value_extraction:", create_processor_job_details_key_value_extraction) -create_processor_response = aiservicedocument_client.create_processor_job_and_wait_for_state( - create_processor_job_details=create_processor_job_details_key_value_extraction, - wait_for_states=[oci.ai_document.models.ProcessorJob.LIFECYCLE_STATE_SUCCEEDED], - waiter_kwargs={"wait_callback": create_processor_job_callback}) - -print("processor call succeeded with status: {} and request_id: {}.".format(create_processor_response.status, create_processor_response.request_id)) -processor_job: oci.ai_document.models.ProcessorJob = create_processor_response.data -print("create_processor_job_details_key_value_extraction response: ", create_processor_response.data) - -print("Getting defaultObject.json from the output_location") -object_storage_client = oci.object_storage.ObjectStorageClient(config=config) -get_object_response = object_storage_client.get_object(namespace_name=output_location.namespace_name, - bucket_name=output_location.bucket_name, - object_name="{}/{}/{}_{}/results/{}.json".format( - output_location.prefix, processor_job.id, - object_location.namespace_name, - object_location.bucket_name, - object_location.object_name)) -print(str(get_object_response.data.content.decode())) \ No newline at end of file diff --git a/ai_services/document_understanding/python/object_storage_document_classification_processor_job_demo.py b/ai_services/document_understanding/python/object_storage_document_classification_processor_job_demo.py new file mode 100644 index 00000000..2da5c2b2 --- /dev/null +++ b/ai_services/document_understanding/python/object_storage_document_classification_processor_job_demo.py @@ -0,0 +1,128 @@ +# coding: utf-8 +# Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved. +# This software is dual-licensed to you under the Universal Permissive License (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl or Apache License 2.0 as shown at http://www.apache.org/licenses/LICENSE-2.0. You may choose either license. + +########################################################################## +# object_storage_document_classification_processor_job_demo.py +# +# Supports Python 3 +########################################################################## +# Info: +# Object Storage Document Classification Processor Job creation using OCI AI Document Understanding service. +# +########################################################################## +# Application Command line(no parameter needed) +# python object_storage_document_classification_processor_job_demo.py +########################################################################## + +""" +This sample script creates an asynchronous processor job to perform document classification on document_classification_demo.pdf +(found in the resources folder), which is expected to be found in the specified object storage bucket. Successful execution of this +sample will create results in object storage as configured under the output_location variable. + +Review and update the instructions following "TODO" to configure the script for your OCI tenancy. +The test document will be uploaded to the object storage bucket specified by 'bucketname' and 'object_location'. + +This script is designed to be executed on a user's computer that is configured with Python 3 or later. The user's computer +must also be configured with a config file as described in README.md to authenticate with the OCI tenancy used to execute +the SDK requests in this script. The authentication config file used by this script will be sourced from the default location (~/.oci/config) +and the default profile in the config file will be used. + +The Document Understanding service region invoked by this example is represented by an endpoint url defined by the variable 'endpoint'. +""" + +import oci +import uuid +import base64 +import os + +# PREPARE TO AUTHENTICATE WITH THE SDK ------------------------------------------------------------ +# Obtain the authentication configuration from the local file +config = oci.config.from_file() +# To use a specific profile in the config file, use these instructions instead: +""" +CONFIG_PROFILE = "doctest_profile" # specify the name of the profile in the config file +config = oci.config.from_file('~/.oci/config', CONFIG_PROFILE) +""" + +# UPLOAD THE DEMO DOCUMENT TO OBJECT STORAGE ------------------------------------------------------------ +# Specify the name of the file to be uploaded +samplefile = "resources/document_classification_demo.pdf" + +# Specify the name of the bucket where the file will be stored +# TODO - set the bucket name from your tenancy +bucketname = "postman" # e.g. "demo-documents" + +# Setup parameters need to upload the sample file +object_storage = oci.object_storage.ObjectStorageClient(config) +namespace = object_storage.get_namespace().data +filename = os.path.basename(samplefile) + +# Upload the sample file to the specified object storage bucket. The document will be overwritten if it alrady exists. +with open(samplefile, 'rb') as f: + obj = object_storage.put_object(namespace,bucketname,filename,f) + print(f'"{filename}" uploaded to bucket "{bucketname}"') + +# CONFIGURE THE JOB ------------------------------------------------------------ +# Specify Compartment where processor job will be created +# TODO - set the compartment ID from your tenancy +COMPARTMENT_ID = "ocid1.compartment.oc1..aaaaaaaag2hxw73h2kz6nkqfs4yvt6sdufhko7ufc367kfgxtcwrcwz5xsgq" # e.g. "ocid1.compartment.oc1..aaaaaaaar3nriwuw4nmqbhe7alu3ou76vgujnq3cvjwjpcv2iivfbadcqfna"; + +def create_processor_job_callback(times_called, response): + print("Waiting for processor lifecycle state to go into succeeded state:", response.data) + +# Initialize client service_endpoint is optional, if not provided then client will point to region specified in oci config +# TODO - set the Document Understanding endpoint for the desired region. See https://docs.oracle.com/en-us/iaas/api/#/en/document-understanding/20221109/ +endpoint = "https://document.aiservice.us-sanjose-1.oci.oraclecloud.com" +aiservicedocument_client = oci.ai_document.AIServiceDocumentClientCompositeOperations(oci.ai_document.AIServiceDocumentClient(config=config, service_endpoint=endpoint)) + +# Specify the Document Classification feature +document_classification_feature = oci.ai_document.models.DocumentClassificationFeature() + +# Specify the location where the document to be processed is stored +#TODO - set .namespace_name and .object_name for your tenancy +object_location = oci.ai_document.models.ObjectLocation() +object_location.namespace_name = "axcqscmddjqn" # e.g. "axk2tfhlrens" +object_location.bucket_name = bucketname +object_location.object_name = filename + +# Setup the output location where processor job results will be created +# TODO - set .namespace_name and .prefix for your tenancy +output_location = oci.ai_document.models.OutputLocation() +output_location.namespace_name = "axcqscmddjqn" # e.g. "axk2tfhlrens" +output_location.bucket_name = bucketname +output_location.prefix = "results-python" + +# Assemble the job details +create_processor_job_details_document_classification_extraction = oci.ai_document.models.CreateProcessorJobDetails( + display_name=str(uuid.uuid4()), + compartment_id=COMPARTMENT_ID, + input_location=oci.ai_document.models.ObjectStorageLocations(object_locations=[object_location]), + output_location=output_location, + processor_config=oci.ai_document.models.GeneralProcessorConfig(features=[document_classification_feature])) + +# EXECUTE THE JOB ------------------------------------------------------------ +# Initiate the processor job status and wait for it to succeed +print("Calling create_processor with create_processor_job_details_document_classification_extraction:", create_processor_job_details_document_classification_extraction, "\n\rWaiting for processor job to finish...") +create_processor_response = aiservicedocument_client.create_processor_job_and_wait_for_state( + create_processor_job_details=create_processor_job_details_document_classification_extraction, + wait_for_states=[oci.ai_document.models.ProcessorJob.LIFECYCLE_STATE_SUCCEEDED], + waiter_kwargs={"wait_callback": create_processor_job_callback}) + +# Get the processor job status and display it +print("Processor job succeeded with status: {} and request_id: {}.".format(create_processor_response.status, create_processor_response.request_id)) +processor_job: oci.ai_document.models.ProcessorJob = create_processor_response.data +print("Job response:\n\r", create_processor_response.data) + +# Get the results from the object storage output location and display it +print("Getting result json from the output_location") +object_storage_client = oci.object_storage.ObjectStorageClient(config=config) +get_object_response = object_storage_client.get_object(namespace_name=output_location.namespace_name, + bucket_name=output_location.bucket_name, + object_name="{}/{}/{}_{}/results/{}.json".format( + output_location.prefix, processor_job.id, + object_location.namespace_name, + object_location.bucket_name, + object_location.object_name)) +print(str(get_object_response.data.content.decode())) +print("You can also access the results in the specified output location under the job id: ", processor_job.id) \ No newline at end of file diff --git a/ai_services/document_understanding/python/object_storage_key_value_extraction_processor_job_demo.py b/ai_services/document_understanding/python/object_storage_key_value_extraction_processor_job_demo.py new file mode 100644 index 00000000..c923b6b1 --- /dev/null +++ b/ai_services/document_understanding/python/object_storage_key_value_extraction_processor_job_demo.py @@ -0,0 +1,150 @@ +# coding: utf-8 +# Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved. +# This software is dual-licensed to you under the Universal Permissive License (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl or Apache License 2.0 as shown at http://www.apache.org/licenses/LICENSE-2.0. You may choose either license. + +########################################################################## +# object_storage_key_value_extraction_processor_job_demo.py +# +# Supports Python 3 +########################################################################## +# Info: +# Object Storage Key Value extraction Processor Job creation using OCI AI Document Understanding service. +# +########################################################################## +# Application Command line(no parameter needed) +# python object_storage_key_value_extraction_processor_job_demo.py +########################################################################## + +""" +This sample script creates an asynchronous processor job to perform key-value extraction on key_value_extraction_demo.pdf +(found in the resources folder), which is expected to be found in the specified object storage bucket. Successful execution of this +sample will create results in object storage as configured under the output_location variable. + +Review and update the instructions following "TODO" to configure the script for your OCI tenancy. +The test document will be uploaded to the object storage bucket specified by 'bucketname' and 'object_location'. + +This script is designed to be executed on a user's computer that is configured with Python 3 or later. The user's computer +must also be configured with a config file as described in README.md to authenticate with the OCI tenancy used to execute +the SDK requests in this script. The authentication config file used by this script will be sourced from the default location (~/.oci/config) +and the default profile in the config file will be used. + +The Document Understanding service region invoked by this example is represented by an endpoint url defined by the variable 'endpoint'. +receipt.jpg +""" +import oci +import uuid +import base64 +import os + +# PREPARE TO AUTHENTICATE WITH THE SDK ------------------------------------------------------------ +# Obtain the authentication configuration from the local file +config = oci.config.from_file() +# To use a specific profile in the config file, use these instructions instead: +""" +CONFIG_PROFILE = "doctest_profile" # specify the name of the profile in the config file +config = oci.config.from_file('~/.oci/config', CONFIG_PROFILE) +""" + +# UPLOAD THE DEMO DOCUMENT TO OBJECT STORAGE ------------------------------------------------------------ +# Specify the name of the file to be uploaded +samplefile = "resources/key_value_extraction_demo.pdf" + +# Specify the name of the bucket where the file will be stored +# TODO - set the bucket name from your tenancy +bucketname = "postman" # e.g. "demo-documents" + +# Setup parameters need to upload the sample file +object_storage = oci.object_storage.ObjectStorageClient(config) +namespace = object_storage.get_namespace().data +filename = os.path.basename(samplefile) + +# Upload the sample file to the specified object storage bucket. The document will be overwritten if it alrady exists. +with open(samplefile, 'rb') as f: + obj = object_storage.put_object(namespace,bucketname,filename,f) + print(f'"{filename}" uploaded to bucket "{bucketname}"') + +# CONFIGURE THE JOB ------------------------------------------------------------ +# Specify Compartment where processor job will be created +# TODO - set the compartment ID from your tenancy +COMPARTMENT_ID = "ocid1.compartment.oc1..aaaaaaaag2hxw73h2kz6nkqfs4yvt6sdufhko7ufc367kfgxtcwrcwz5xsgq" # e.g. "ocid1.compartment.oc1..aaaaaaaar3nriwuw4nmqbhe7alu3ou76vgujnq3cvjwjpcv2iivfbadcqfna"; + +def create_processor_job_callback(times_called, response): + print("Waiting for processor lifecycle state to go into succeeded state:", response.data) + +# Initialize client service_endpoint is optional, if not provided then client will point to region specified in oci config +# TODO - set the Document Understanding endpoint for the desired region. See https://docs.oracle.com/en-us/iaas/api/#/en/document-understanding/20221109/ +endpoint = "https://document.aiservice.us-sanjose-1.oci.oraclecloud.com" +aiservicedocument_client = oci.ai_document.AIServiceDocumentClientCompositeOperations(oci.ai_document.AIServiceDocumentClient(config=config, service_endpoint=endpoint)) + +# Specify the Key-Value extraction feature +key_value_extraction_feature = oci.ai_document.models.DocumentKeyValueExtractionFeature() + +# Specify the location where the document to be processed is stored +#TODO - set .namespace_name and .object_name for your tenancy +object_location = oci.ai_document.models.ObjectLocation() +object_location.namespace_name = "axcqscmddjqn" # e.g. "axk2tfhlrens" +object_location.bucket_name = bucketname +object_location.object_name = "key_value_extraction_demo.pdf" # e.g "invoice.pdf" + +# Setup the output location where processor job results will be created +# TODO - set .namespace_name and .prefix for your tenancy +output_location = oci.ai_document.models.OutputLocation() +output_location.namespace_name = "axcqscmddjqn" # e.g. "axk2tfhlrens" +output_location.bucket_name = bucketname +output_location.prefix = "results-python" + +# Assemble the job details +create_processor_job_details_key_value_extraction = oci.ai_document.models.CreateProcessorJobDetails( + display_name=str(uuid.uuid4()), + compartment_id=COMPARTMENT_ID, + input_location=oci.ai_document.models.ObjectStorageLocations(object_locations=[object_location]), + output_location=output_location, + processor_config=oci.ai_document.models.GeneralProcessorConfig(features=[key_value_extraction_feature])) +""" +print("Calling create_processor with create_processor_job_details_key_value_extraction:", create_processor_job_details_key_value_extraction) +create_processor_response = aiservicedocument_client.create_processor_job_and_wait_for_state( + create_processor_job_details=create_processor_job_details_key_value_extraction, + wait_for_states=[oci.ai_document.models.ProcessorJob.LIFECYCLE_STATE_SUCCEEDED], + waiter_kwargs={"wait_callback": create_processor_job_callback}) + +print("processor call succeeded with status: {} and request_id: {}.".format(create_processor_response.status, create_processor_response.request_id)) +processor_job: oci.ai_document.models.ProcessorJob = create_processor_response.data +print("create_processor_job_details_key_value_extraction response: ", create_processor_response.data) + +print("Getting result json from the output_location") +object_storage_client = oci.object_storage.ObjectStorageClient(config=config) +get_object_response = object_storage_client.get_object(namespace_name=output_location.namespace_name, + bucket_name=output_location.bucket_name, + object_name="{}/{}/{}_{}/results/{}.json".format( + output_location.prefix, processor_job.id, + object_location.namespace_name, + object_location.bucket_name, + object_location.object_name)) + +print(str(get_object_response.data.content.decode())) +""" +# EXECUTE THE JOB ------------------------------------------------------------ +# Initiate the processor job status and wait for it to succeed +print("Calling create_processor with create_processor_job_details_key_value_extraction:", create_processor_job_details_key_value_extraction, "\n\rWaiting for processor job to finish...") +create_processor_response = aiservicedocument_client.create_processor_job_and_wait_for_state( + create_processor_job_details=create_processor_job_details_key_value_extraction, + wait_for_states=[oci.ai_document.models.ProcessorJob.LIFECYCLE_STATE_SUCCEEDED], + waiter_kwargs={"wait_callback": create_processor_job_callback}) + +# Get the processor job status and display it +print("Processor job succeeded with status: {} and request_id: {}.".format(create_processor_response.status, create_processor_response.request_id)) +processor_job: oci.ai_document.models.ProcessorJob = create_processor_response.data +print("Job response:\n\r", create_processor_response.data) + +# Get the results from the object storage output location and display it +print("Getting result json from the output_location") +object_storage_client = oci.object_storage.ObjectStorageClient(config=config) +get_object_response = object_storage_client.get_object(namespace_name=output_location.namespace_name, + bucket_name=output_location.bucket_name, + object_name="{}/{}/{}_{}/results/{}.json".format( + output_location.prefix, processor_job.id, + object_location.namespace_name, + object_location.bucket_name, + object_location.object_name)) +print(str(get_object_response.data.content.decode())) +print("You can also access the results in the specified output location under the job id: ", processor_job.id) \ No newline at end of file diff --git a/ai_services/document_understanding/python/object_storage_table_extraction_processor_job_demo.py b/ai_services/document_understanding/python/object_storage_table_extraction_processor_job_demo.py new file mode 100644 index 00000000..34f9c1f2 --- /dev/null +++ b/ai_services/document_understanding/python/object_storage_table_extraction_processor_job_demo.py @@ -0,0 +1,128 @@ +# coding: utf-8 +# Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved. +# This software is dual-licensed to you under the Universal Permissive License (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl or Apache License 2.0 as shown at http://www.apache.org/licenses/LICENSE-2.0. You may choose either license. + +########################################################################## +# object_storage_table_extraction_processor_job_demo.py +# +# Supports Python 3 +########################################################################## +# Info: +# Object Storage Table extraction Processor Job creation using OCI AI Document Understanding service. +# +########################################################################## +# Application Command line(no parameter needed) +# python object_storage_table_extraction_processor_job_demo.py +########################################################################## + +""" +This sample script creates an asynchronous processor job to perform table extraction on table_extraction_demo.pdf +(found in the resources folder), which is expected to be found in the specified object storage bucket. Successful execution of this +sample will create results in object storage as configured under the output_location variable. + +Review and update the instructions following "TODO" to configure the script for your OCI tenancy. +The test document will be uploaded to the object storage bucket specified by 'bucketname' and 'object_location'. + +This script is designed to be executed on a user's computer that is configured with Python 3 or later. The user's computer +must also be configured with a config file as described in README.md to authenticate with the OCI tenancy used to execute +the SDK requests in this script. The authentication config file used by this script will be sourced from the default location (~/.oci/config) +and the default profile in the config file will be used. + +The Document Understanding service region invoked by this example is represented by an endpoint url defined by the variable 'endpoint'. +""" + +import oci +import uuid +import base64 +import os + +# PREPARE TO AUTHENTICATE WITH THE SDK ------------------------------------------------------------ +# Obtain the authentication configuration from the local file +config = oci.config.from_file() +# To use a specific profile in the config file, use these instructions instead: +""" +CONFIG_PROFILE = "doctest_profile" # specify the name of the profile in the config file +config = oci.config.from_file('~/.oci/config', CONFIG_PROFILE) +""" + +# UPLOAD THE DEMO DOCUMENT TO OBJECT STORAGE ------------------------------------------------------------ +# Specify the name of the file to be uploaded +samplefile = "resources/table_extraction_demo.pdf" + +# Specify the name of the bucket where the file will be stored +# TODO - set the bucket name from your tenancy +bucketname = "postman" # e.g. "demo-documents" + +# Setup parameters need to upload the sample file +object_storage = oci.object_storage.ObjectStorageClient(config) +namespace = object_storage.get_namespace().data +filename = os.path.basename(samplefile) + +# Upload the sample file to the specified object storage bucket. The document will be overwritten if it alrady exists. +with open(samplefile, 'rb') as f: + obj = object_storage.put_object(namespace,bucketname,filename,f) + print(f'"{filename}" uploaded to bucket "{bucketname}"') + +# CONFIGURE THE JOB ------------------------------------------------------------ +# Specify Compartment where processor job will be created +# TODO - set the compartment ID from your tenancy +COMPARTMENT_ID = "ocid1.compartment.oc1..aaaaaaaag2hxw73h2kz6nkqfs4yvt6sdufhko7ufc367kfgxtcwrcwz5xsgq" # e.g. "ocid1.compartment.oc1..aaaaaaaar3nriwuw4nmqbhe7alu3ou76vgujnq3cvjwjpcv2iivfbadcqfna"; + +def create_processor_job_callback(times_called, response): + print("Waiting for processor lifecycle state to go into succeeded state:", response.data) + +# Initialize client service_endpoint is optional, if not provided then client will point to region specified in oci config +# TODO - set the Document Understanding endpoint for the desired region. See https://docs.oracle.com/en-us/iaas/api/#/en/document-understanding/20221109/ +endpoint = "https://document.aiservice.us-sanjose-1.oci.oraclecloud.com" +aiservicedocument_client = oci.ai_document.AIServiceDocumentClientCompositeOperations(oci.ai_document.AIServiceDocumentClient(config=config, service_endpoint=endpoint)) + +# Specify the Table Extraction Feature +table_extraction_feature = oci.ai_document.models.DocumentTableExtractionFeature() + +# Specify the location where the document to be processed is stored +#TODO - set .namespace_name and .object_name for your tenancy +object_location = oci.ai_document.models.ObjectLocation() +object_location.namespace_name = "axcqscmddjqn" # e.g. "axk2tfhlrens" +object_location.bucket_name = bucketname +object_location.object_name = filename + +# Setup the output location where processor job results will be created +# TODO - set .namespace_name and .prefix for your tenancy +output_location = oci.ai_document.models.OutputLocation() +output_location.namespace_name = "axcqscmddjqn" # e.g. "axk2tfhlrens" +output_location.bucket_name = bucketname +output_location.prefix = "results-python" + +# Assemble the job details +create_processor_job_details_table_extraction = oci.ai_document.models.CreateProcessorJobDetails( + display_name=str(uuid.uuid4()), + compartment_id=COMPARTMENT_ID, + input_location=oci.ai_document.models.ObjectStorageLocations(object_locations=[object_location]), + output_location=output_location, + processor_config=oci.ai_document.models.GeneralProcessorConfig(features=[table_extraction_feature])) + +# EXECUTE THE JOB ------------------------------------------------------------ +# Initiate the processor job status and wait for it to succeed +print("Calling create_processor with create_processor_job_details_table_extraction:", create_processor_job_details_table_extraction, "\n\rWaiting for processor job to finish...") +create_processor_response = aiservicedocument_client.create_processor_job_and_wait_for_state( + create_processor_job_details=create_processor_job_details_table_extraction, + wait_for_states=[oci.ai_document.models.ProcessorJob.LIFECYCLE_STATE_SUCCEEDED], + waiter_kwargs={"wait_callback": create_processor_job_callback}) + +# Get the processor job status and display it +print("Processor job succeeded with status: {} and request_id: {}.".format(create_processor_response.status, create_processor_response.request_id)) +processor_job: oci.ai_document.models.ProcessorJob = create_processor_response.data +print("Job response:\n\r", create_processor_response.data) + +# Get the results from the object storage output location and display it +print("Getting result json from the output_location") +object_storage_client = oci.object_storage.ObjectStorageClient(config=config) +get_object_response = object_storage_client.get_object(namespace_name=output_location.namespace_name, + bucket_name=output_location.bucket_name, + object_name="{}/{}/{}_{}/results/{}.json".format( + output_location.prefix, processor_job.id, + object_location.namespace_name, + object_location.bucket_name, + object_location.object_name)) +print(str(get_object_response.data.content.decode())) +print("You can also access the results in the specified output location under the job id: ", processor_job.id) \ No newline at end of file diff --git a/ai_services/document_understanding/python/object_storage_text_extraction_processor_job_demo.py b/ai_services/document_understanding/python/object_storage_text_extraction_processor_job_demo.py new file mode 100644 index 00000000..f9aed7a9 --- /dev/null +++ b/ai_services/document_understanding/python/object_storage_text_extraction_processor_job_demo.py @@ -0,0 +1,130 @@ +# coding: utf-8 +# Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved. +# This software is dual-licensed to you under the Universal Permissive License (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl or Apache License 2.0 as shown at http://www.apache.org/licenses/LICENSE-2.0. You may choose either license. + +########################################################################## +# object_storage_text_extraction_processor_job_demo.py +# +# Supports Python 3 +########################################################################## +# Info: +# Object Storage Text extraction Processor Job creation using OCI AI Document Understanding service. +# +########################################################################## +# Application Command line(no parameter needed) +# python object_storage_text_extraction_processor_job_demo.py +########################################################################## + +""" +This sample script creates an asynchronous processor job to perform text extraction on text_extraction_demo.jpg +(found in the resources folder), which is expected to be found in the specified object storage bucket. Successful execution of this +sample will create results in object storage as configured under the output_location variable. + +Review and update the instructions following "TODO" to configure the script for your OCI tenancy. +The test document will be uploaded to the object storage bucket specified by 'bucketname' and 'object_location'. + +This script is designed to be executed on a user's computer that is configured with Python 3 or later. The user's computer +must also be configured with a config file as described in README.md to authenticate with the OCI tenancy used to execute +the SDK requests in this script. The authentication config file used by this script will be sourced from the default location (~/.oci/config) +and the default profile in the config file will be used. + +The Document Understanding service region invoked by this example is represented by an endpoint url defined by the variable 'endpoint'. +""" + +import oci +import uuid +import base64 +import os + +# PREPARE TO AUTHENTICATE WITH THE SDK ------------------------------------------------------------ +# Obtain the authentication configuration from the local file +config = oci.config.from_file() +# To use a specific profile in the config file, use these instructions instead: +""" +CONFIG_PROFILE = "doctest_profile" # specify the name of the profile in the config file +config = oci.config.from_file('~/.oci/config', CONFIG_PROFILE) +""" + +# UPLOAD THE DEMO DOCUMENT TO OBJECT STORAGE ------------------------------------------------------------ +# Specify the name of the file to be uploaded +samplefile = "resources/text_extraction_demo.jpg" + +# Specify the name of the bucket where the file will be stored +# TODO - set the bucket name from your tenancy +bucketname = "postman" # e.g. "demo-documents" + +# Setup parameters need to upload the sample file +object_storage = oci.object_storage.ObjectStorageClient(config) +namespace = object_storage.get_namespace().data +filename = os.path.basename(samplefile) + +# Upload the sample file to the specified object storage bucket. The document will be overwritten if it alrady exists. +with open(samplefile, 'rb') as f: + obj = object_storage.put_object(namespace,bucketname,filename,f) + print(f'"{filename}" uploaded to bucket "{bucketname}"') + +# CONFIGURE THE JOB ------------------------------------------------------------ +# Specify Compartment where processor job will be created +# TODO - set the compartment ID from your tenancy +COMPARTMENT_ID = "ocid1.compartment.oc1..aaaaaaaag2hxw73h2kz6nkqfs4yvt6sdufhko7ufc367kfgxtcwrcwz5xsgq" # e.g. "ocid1.compartment.oc1..aaaaaaaar3nriwuw4nmqbhe7alu3ou76vgujnq3cvjwjpcv2iivfbadcqfna"; + +def create_processor_job_callback(times_called, response): + print("Waiting for processor lifecycle state to go into succeeded state:", response.data) + +# Initialize client service_endpoint is optional, if not provided then client will point to region specified in oci config +# TODO - set the Document Understanding endpoint for the desired region. See https://docs.oracle.com/en-us/iaas/api/#/en/document-understanding/20221109/ +endpoint = "https://document.aiservice.us-sanjose-1.oci.oraclecloud.com" +aiservicedocument_client = oci.ai_document.AIServiceDocumentClientCompositeOperations(oci.ai_document.AIServiceDocumentClient(config=config, service_endpoint=endpoint)) + +# Specify the text extraction eature +text_extraction_feature_searchable_pdf = oci.ai_document.models.DocumentTextExtractionFeature() +# Additionally, specify the option to generate a searchable PDF +text_extraction_feature_searchable_pdf.generate_searchable_pdf = False # use True to generate the PDF + +# Specify the location where the document to be processed is stored +#TODO - set .namespace_name and .object_name for your tenancy +object_location = oci.ai_document.models.ObjectLocation() +object_location.namespace_name = "axcqscmddjqn" # e.g. "axk2tfhlrens" +object_location.bucket_name = bucketname +object_location.object_name = filename + +# Setup the output location where processor job results will be created +# TODO - set .namespace_name and .prefix for your tenancy +output_location = oci.ai_document.models.OutputLocation() +output_location.namespace_name = "axcqscmddjqn" # e.g. "axk2tfhlrens" +output_location.bucket_name = bucketname +output_location.prefix = "results-python" + +# Assemble the job details +create_processor_job_details_text_extraction = oci.ai_document.models.CreateProcessorJobDetails( + display_name=str(uuid.uuid4()), + compartment_id=COMPARTMENT_ID, + input_location=oci.ai_document.models.ObjectStorageLocations(object_locations=[object_location]), + output_location=output_location, + processor_config=oci.ai_document.models.GeneralProcessorConfig(features=[text_extraction_feature_searchable_pdf])) + +# EXECUTE THE JOB ------------------------------------------------------------ +# Initiate the processor job status and wait for it to succeed +print("Calling create_processor with create_processor_job_details_text_extraction:", create_processor_job_details_text_extraction, "\n\rWaiting for processor job to finish...") +create_processor_response = aiservicedocument_client.create_processor_job_and_wait_for_state( + create_processor_job_details=create_processor_job_details_text_extraction, + wait_for_states=[oci.ai_document.models.ProcessorJob.LIFECYCLE_STATE_SUCCEEDED], + waiter_kwargs={"wait_callback": create_processor_job_callback}) + +# Get the processor job status and display it +print("Processor job succeeded with status: {} and request_id: {}.".format(create_processor_response.status, create_processor_response.request_id)) +processor_job: oci.ai_document.models.ProcessorJob = create_processor_response.data +print("Job response:\n\r", create_processor_response.data) + +# Get the results from the object storage output location and display it +print("Getting result json from the output_location") +object_storage_client = oci.object_storage.ObjectStorageClient(config=config) +get_object_response = object_storage_client.get_object(namespace_name=output_location.namespace_name, + bucket_name=output_location.bucket_name, + object_name="{}/{}/{}_{}/results/{}.json".format( + output_location.prefix, processor_job.id, + object_location.namespace_name, + object_location.bucket_name, + object_location.object_name)) +print(str(get_object_response.data.content.decode())) +print("You can also access the results in the specified output location under the job id: ", processor_job.id) \ No newline at end of file diff --git a/ai_services/document_understanding/python/readme.md b/ai_services/document_understanding/python/readme.md index a5b2782e..86b6d733 100644 --- a/ai_services/document_understanding/python/readme.md +++ b/ai_services/document_understanding/python/readme.md @@ -2,15 +2,21 @@ ## Introduction -Oracle Cloud Infrastructure provides a number of Software Development Kits (SDKs) to facilitate development of custom solutions. SDKs allow you to build and deploy apps that integrate with Oracle Cloud Infrastructure services. Each SDK also includes tools and artifacts you need to develop an app, such as code samples and documentation. In addition, if you want to contribute to the development of the SDKs, they are all open source and available on GitHub. +Oracle Cloud Infrastructure provides a number of programming language Software Development Kits (SDKs) to facilitate application integration and development of custom solutions. SDKs allow you to build and deploy apps that integrate with Oracle Cloud Infrastructure services. Each SDK also includes tools and artifacts you need to develop an app, such as code samples and documentation. In addition, if you want to contribute to the development of the SDKs, they are all open source and available on GitHub. You can invoke OCI Document Understanding capabilities through the OCI SDKs. -## Pre-requisites: +## Prerequisites: + +### Ensure Document Understanding is configured in your OCI tenancy +Review this documentation to setup policies that allow users to access the document Understanding APIs: +* [Policy to Grant Users Access to Document Understanding APIs](https://docs.oracle.com/en-us/iaas/Content/document-understanding/using/about_document-understanding_policies.htm#policy_users) + +It's also a good idea to verify your user account can process documents using the OCI Console for your tenancy. ### Set up config file -You need to set up an API Signing Key and a configuration file so that the SDK can find the credentials needed to connect to your OCI tenancy and use the document understanding capabilities. +You need to set up an API Signing Key and a configuration file so that the SDK can find the credentials needed to connect to your OCI tenancy and use the Document Understanding capabilities. If you have never done this before, you may want to follow the steps described in the [OCI Document Understanding Workshop - Lab 4](https://github.com/oracle-samples/oci-data-science-ai-samples/tree/master/labs/ai-document-understanding/workshops/4-python-sdk). @@ -18,6 +24,10 @@ Other related documents: * [Generating API KEY](https://docs.oracle.com/en-us/iaas/Content/API/Concepts/apisigningkey.htm) * [SDK and CLI Configuration File](https://docs.oracle.com/en-us/iaas/Content/API/Concepts/sdkconfig.htm#SDK_and_CLI_Configuration_File) +After completion, you should have following 2 files in your ~/.oci directory + +1. A config file (where key_file refers to private key for your user and tenancy: e.g. key_file=~/.oci/oci_api_key.pem) +2. A private key file, e.g. oci_api_key.pem ### Python requirements @@ -55,17 +65,151 @@ Once you have activated your environment, install the OCI SDK by running: > pip install oci - -## Samples - -Samples are organized by feature, and each folder contains sample code for either an inline input document or an input document from object storage. - -Features include: - -**text_extraction** - -**table_extraction** - -**document_classification** - -**key_value_extraction** \ No newline at end of file +(If you are on a VPN, ensure it allows you to connect to pypi.org. You might need to disconnect from VPN.) + +## Execute the sample scripts +### Change directory to this project location +In the python environment that you activated, change the directory to the location where this project resides on your local machine. + +``` +cd /python +``` + +### Edit the scripts to work with your OCI tenancy + +The comments in each script contain instructions on what values need to be updated for your specific OCI tenancy. +This includes things like compartment ID, regional service endpoint, and object storage locations. + +### Execute any or all of the python sample scripts using these commands + +Synchronous API using an inline source document: +``` +python sync_inline_text_extraction_demo.py +``` + +Asynchronous API using an inline document source: +``` +python inline_text_extraction_processor_job_demo.py +python inline_table_extraction_processor_job_demo.py +python inline_key_value_extraction_processor_job_demo.py +python inline_document_classification_processor_job_demo.py +``` + +Asynchronous API using object storage as the document source: +``` +python object_storage_text_extraction_processor_job_demo.py +python object_storage_table_extraction_processor_job_demo.py +python object_storage_key_value_extraction_processor_job_demo.py +python object_storage_document_classification_processor_job_demo.py +``` + +### When running each of above scripts, you will see the request and the response similar to this example from inline_document_classification_processor_job_demo.py: +Note that the output contains the base64 encoded document, which is REMOVED here for brevity +``` +python inline_document_classification_processor_job_demo.py +Calling create_processor with create_processor_job_details_document_classification_extraction: { + "compartment_id": "ocid1.compartment.oc1..aaaaaaaag2hxw73h2kz6nkqfs4yvt6sdufhko7ufc367kfgxtcwrcwz5xsgq", + "display_name": "8a868eb6-83a6-4ec1-a576-951fc4bf6752", + "input_location": { + "data": "--REMOVED--", + "source_type": "INLINE_DOCUMENT_CONTENT" + }, + "output_location": { + "bucket_name": "postman", + "namespace_name": "axcqscmddjqn", + "prefix": "results-python" + }, + "processor_config": { + "document_type": null, + "features": [ + { + "feature_type": "DOCUMENT_CLASSIFICATION", + "max_results": null, + "model_id": null, + "tenancy_id": null + } + ], + "is_zip_output_enabled": null, + "language": null, + "processor_type": "GENERAL" + } +} +Waiting for processor job to finish... +processor call succeeded with status: 200 and request_id: AAEF79E1FC7447FE88E4E2A44D932B05/CD7BEAB837D2CC81FAD506CFB98BEFA7/4340A8CCA1D3B15BC3EB41DE9BD1FFBA. +create_processor_job_details_document_classification_extraction response: { + "compartment_id": "ocid1.compartment.oc1..aaaaaaaag2hxw73h2kz6nkqfs4yvt6sdufhko7ufc367kfgxtcwrcwz5xsgq", + "display_name": "8a868eb6-83a6-4ec1-a576-951fc4bf6752", + "id": "ocid1.aidocumentprocessorjob.oc1.us-sanjose-1.amaaaaaa3nkmftyasfwqfleictfbm2bwlwz6kwklne3iqcezyl6asxaier2q", + "input_location": null, + "lifecycle_details": null, + "lifecycle_state": "SUCCEEDED", + "output_location": { + "bucket_name": "postman", + "namespace_name": "axcqscmddjqn", + "prefix": "results-python" + }, + "percent_complete": 100.0, + "processor_config": { + "document_type": null, + "features": [ + { + "feature_type": "DOCUMENT_CLASSIFICATION", + "max_results": null, + "model_id": null, + "tenancy_id": null + } + ], + "is_zip_output_enabled": false, + "language": null, + "processor_type": "GENERAL" + }, + "time_accepted": "2024-01-10T23:12:59.520000+00:00", + "time_finished": "2024-01-10T23:12:59.520000+00:00", + "time_started": "2024-01-10T23:12:59.520000+00:00" +} +Getting defaultObject.json from the output_location +{ + "documentMetadata" : { + "pageCount" : 1, + "mimeType" : "application/pdf" + }, + "pages" : [ { + "pageNumber" : 1, + "dimensions" : null, + "detectedDocumentTypes" : [ { + "documentType" : "RESUME", + "confidence" : 0.99459565 + }, { + "documentType" : "PASSPORT", + "confidence" : 8.510603E-4 + }, { + "documentType" : "OTHERS", + "confidence" : 7.4783183E-4 + }, { + "documentType" : "TAX_FORM", + "confidence" : 7.022046E-4 + }, { + "documentType" : "INVOICE", + "confidence" : 6.5188756E-4 + } ], + "detectedLanguages" : null, + "words" : null, + "lines" : null, + "tables" : null, + "documentFields" : null + } ], + "detectedDocumentTypes" : [ { + "documentType" : "RESUME", + "confidence" : 0.99459565 + } ], + "detectedLanguages" : null, + "documentClassificationModelVersion" : "1.6.43", + "languageClassificationModelVersion" : null, + "textExtractionModelVersion" : null, + "keyValueExtractionModelVersion" : null, + "tableExtractionModelVersion" : null, + "errors" : null, + "searchablePdf" : null +} +You can also view the results in the specified output location under the job id: ocid1.aidocumentprocessorjob.oc1.us-sanjose-1.amaaaaaa3nkmftyasfwqfleictfbm2bwlwz6kwklne3iqcezyl6asxaier2q +``` diff --git a/ai_services/document_understanding/python/requirements-local.txt b/ai_services/document_understanding/python/requirements-local.txt new file mode 100644 index 00000000..20780eac --- /dev/null +++ b/ai_services/document_understanding/python/requirements-local.txt @@ -0,0 +1,3 @@ +--extra-index-url="https://artifactory.oci.oraclecorp.com/api/pypi/global-dev-pypi/simple" +--trusted-host="artifactory.oci.oraclecorp.com" +oci==2.108.0 diff --git a/ai_services/document_understanding/python/requirements.txt b/ai_services/document_understanding/python/requirements.txt new file mode 100644 index 00000000..b48ac0cf --- /dev/null +++ b/ai_services/document_understanding/python/requirements.txt @@ -0,0 +1 @@ +oci==2.108.0 diff --git a/ai_services/document_understanding/python/resources/document_classification_demo.pdf b/ai_services/document_understanding/python/resources/document_classification_demo.pdf index a6293359..c9069fc7 100644 Binary files a/ai_services/document_understanding/python/resources/document_classification_demo.pdf and b/ai_services/document_understanding/python/resources/document_classification_demo.pdf differ diff --git a/ai_services/document_understanding/python/resources/table_demo.jpg b/ai_services/document_understanding/python/resources/table_demo.jpg deleted file mode 100644 index c52975a6..00000000 Binary files a/ai_services/document_understanding/python/resources/table_demo.jpg and /dev/null differ diff --git a/ai_services/document_understanding/python/resources/table_extraction_demo.pdf b/ai_services/document_understanding/python/resources/table_extraction_demo.pdf new file mode 100644 index 00000000..00890bae Binary files /dev/null and b/ai_services/document_understanding/python/resources/table_extraction_demo.pdf differ diff --git a/ai_services/document_understanding/python/resources/text_extraction_demo.jpg b/ai_services/document_understanding/python/resources/text_extraction_demo.jpg new file mode 100644 index 00000000..196d7ce6 Binary files /dev/null and b/ai_services/document_understanding/python/resources/text_extraction_demo.jpg differ diff --git a/ai_services/document_understanding/python/resources/text_extraction_demo.pdf b/ai_services/document_understanding/python/resources/text_extraction_demo.pdf deleted file mode 100644 index 1a0b9383..00000000 Binary files a/ai_services/document_understanding/python/resources/text_extraction_demo.pdf and /dev/null differ diff --git a/ai_services/document_understanding/python/sync_inline_text_extraction_demo.py b/ai_services/document_understanding/python/sync_inline_text_extraction_demo.py new file mode 100644 index 00000000..d791ef6b --- /dev/null +++ b/ai_services/document_understanding/python/sync_inline_text_extraction_demo.py @@ -0,0 +1,78 @@ +# coding: utf-8 +# Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved. +# This software is dual-licensed to you under the Universal Permissive License (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl or Apache License 2.0 as shown at http://www.apache.org/licenses/LICENSE-2.0. You may choose either license. + +########################################################################## +# sync_inline_text_extraction_demo.py +# +# Supports Python 3 +########################################################################## +# Info: +# Inline Text extraction using synchronous Analyze Document method of OCI AI Document Understanding service. +# +########################################################################## +# Application Command line(no parameter needed) +# python sync_inline_text_extraction_demo.py +########################################################################## + +""" +This sample script executes a synchronous request for text extraction on text_extraction_demo.jpg +(found in the resources folder), which is supplied inline as a base64 encoded string. +Because this is a synchronous request, the results are returned in the response. +Object storage is not used at all for this request. + +This script is designed to be executed on a user's computer that is configured with Python 3 +or later. The user's computer must also be configured with a config file as described in +README.md to authenticate with the OCI tenancy used to execute the SDK requests in this script. +The authentication config file used by this script will be sourced from the default location +(~/.oci/config) and the default profile in the config file will be used. + +The Document Understanding service region invoked by this example is represented by an endpoint +url defined by the variable 'endpoint' +""" + +import oci +import uuid +import base64 + +# CONFIGURE THE REQUEST ------------------------------------------------------------ +# Auth Config +# Initialize the service client with the default profile in the config file +config = oci.config.from_file() +# To use a specific profile in the config file, use these instructions instead: +""" +CONFIG_PROFILE = "doctest_profile" # specify the name of the profile in the config file +config = oci.config.from_file('~/.oci/config', CONFIG_PROFILE) +""" + +# Compartment where processor job will be created +# TODO - set the compartment ID from your tenancy +COMPARTMENT_ID = "ocid1.compartment.oc1..aaaaaaaag2hxw73h2kz6nkqfs4yvt6sdufhko7ufc367kfgxtcwrcwz5xsgq" # e.g. "ocid1.compartment.oc1..aaaaaaaar3nriwuw4nmqbhe7alu3ou76vgujnq3cvjwjpcv2iivfbadcqfna"; + +# Encode the sample document for submitting inline +text_extraction_sample_string = None +with open("resources/text_extraction_demo.jpg", "rb") as document_file: + text_extraction_sample_string = base64.b64encode(document_file.read()).decode('utf-8') + +# Specify the Document Understanding service endpoint +# TODO - specify the Document Understanding endpoint for the desired region. See https://docs.oracle.com/en-us/iaas/api/#/en/document-understanding/20221109/ +endpoint = "https://document.aiservice.us-sanjose-1.oci.oraclecloud.com" +aiservicedocument_client = oci.ai_document.AIServiceDocumentClient(config=config, service_endpoint=endpoint) + +# Specify text extraction is to be performed on the sample document +text_extraction_feature_searchable_pdf = oci.ai_document.models.DocumentTextExtractionFeature() +# Furthermore, specify if a Searchable PDF is to be generated from the text extraction results +text_extraction_feature_searchable_pdf.generate_searchable_pdf = False #set to True to generate a searchable PDF + +# Assemble the request parameters +document_details = oci.ai_document.models.AnalyzeDocumentDetails(features=[text_extraction_feature_searchable_pdf], + document=oci.ai_document.models.InlineDocumentDetails( + data=text_extraction_sample_string), + compartment_id=COMPARTMENT_ID + ) + +# EXECUTE THE REQUEST ------------------------------------------------------------ +print("Calling synchronous api...") +response = aiservicedocument_client.analyze_document(analyze_document_details=document_details) +# Print the response +print("Text extraction result:\n\r", response.data) \ No newline at end of file diff --git a/ai_services/document_understanding/python/table_extraction/object_storage_table_extraction_demo.py b/ai_services/document_understanding/python/table_extraction/object_storage_table_extraction_demo.py deleted file mode 100644 index a4e3daa8..00000000 --- a/ai_services/document_understanding/python/table_extraction/object_storage_table_extraction_demo.py +++ /dev/null @@ -1,86 +0,0 @@ -# coding: utf-8 -# Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved. -# This software is dual-licensed to you under the Universal Permissive License (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl or Apache License 2.0 as shown at http://www.apache.org/licenses/LICENSE-2.0. You may choose either license. - -########################################################################## -# object_storage_table_extraction_demo.py -# -# Supports Python 3 -########################################################################## -# Info: -# Object Storage table extraction Processor Job creation using OCI AI Document Understanding service. -# -########################################################################## -# Application Command line(no parameter needed) -# python object_storage_table_extraction_demo.py -########################################################################## - -""" -This python script provides an example of how to use OCI Document Understanding Service table extraction feature. - -The configuration file used by service clients will be sourced from the default location (~/.oci/config) and the -CONFIG_PROFILE profile will be used. - -The sample attempts to extract table information from a document in object storage. -Successful run of this sample will create job results under object storage configured under output_location variable -""" -import oci -import uuid -import base64 - -# Setup basic variables -# Auth Config -CONFIG_PROFILE = "DEFAULT" -config = oci.config.from_file('~/.oci/config', CONFIG_PROFILE) - -# Compartment where processor job will be created (required) -COMPARTMENT_ID = "" # e.g. "ocid1.compartment.oc1..aaaaaaaae5j73axsja5fnahbn23ilop3ynjkcg77mcvgryddz4pkh2t5ppaq"; - -def create_processor_job_callback(times_called, response): - print("Waiting for processor lifecycle state to go into succeeded state:", response.data) - -aiservicedocument_client = oci.ai_document.AIServiceDocumentClientCompositeOperations(oci.ai_document.AIServiceDocumentClient(config=config)) - -# Table extraction Feature -table_extraction_feature = oci.ai_document.models.DocumentTableExtractionFeature() - -# Setup input location where document being processed is stored. -object_location = oci.ai_document.models.ObjectLocation() -object_location.namespace_name = "" # e.g. "axhh9gizbq5x" -object_location.bucket_name = "" # e.g "demo_examples" -object_location.object_name = "" # e.g "table_demo.jpg" - -# Setup the output location where processor job results will be created -output_location = oci.ai_document.models.OutputLocation() -output_location.namespace_name = "" # e.g. "axk2tfhlrens" -output_location.bucket_name = "" # e.g "output" -output_location.prefix = "" # e.g "demo" - -# Create a processor_job for table_extraction feature -create_processor_job_details_table_extraction = oci.ai_document.models.CreateProcessorJobDetails( - display_name=str(uuid.uuid4()), - compartment_id=COMPARTMENT_ID, - input_location=oci.ai_document.models.ObjectStorageLocations(object_locations=[object_location]), - output_location=output_location, - processor_config=oci.ai_document.models.GeneralProcessorConfig(features=[table_extraction_feature])) - -print("Calling create_processor with create_processor_job_details_table_extraction:", create_processor_job_details_table_extraction) -create_processor_response = aiservicedocument_client.create_processor_job_and_wait_for_state( - create_processor_job_details=create_processor_job_details_table_extraction, - wait_for_states=[oci.ai_document.models.ProcessorJob.LIFECYCLE_STATE_SUCCEEDED], - waiter_kwargs={"wait_callback": create_processor_job_callback}) - -print("processor call succeeded with status: {} and request_id: {}.".format(create_processor_response.status, create_processor_response.request_id)) -processor_job: oci.ai_document.models.ProcessorJob = create_processor_response.data -print("create_processor_job_details_table_extraction response: ", create_processor_response.data) - -print("Getting defaultObject.json from the output_location") -object_storage_client = oci.object_storage.ObjectStorageClient(config=config) -get_object_response = object_storage_client.get_object(namespace_name=output_location.namespace_name, - bucket_name=output_location.bucket_name, - object_name="{}/{}/{}_{}/results/{}.json".format( - output_location.prefix, processor_job.id, - object_location.namespace_name, - object_location.bucket_name, - object_location.object_name)) -print(str(get_object_response.data.content.decode())) \ No newline at end of file diff --git a/ai_services/document_understanding/python/text_extraction/inline_text_extraction_demo.py b/ai_services/document_understanding/python/text_extraction/inline_text_extraction_demo.py deleted file mode 100644 index 5aa74655..00000000 --- a/ai_services/document_understanding/python/text_extraction/inline_text_extraction_demo.py +++ /dev/null @@ -1,82 +0,0 @@ -# coding: utf-8 -# Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved. -# This software is dual-licensed to you under the Universal Permissive License (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl or Apache License 2.0 as shown at http://www.apache.org/licenses/LICENSE-2.0. You may choose either license. - -########################################################################## -# inline_text_extraction_demo.py -# -# Supports Python 3 -########################################################################## -# Info: -# Inline Text Extraction Processor Job creation using OCI AI Document Understanding service. -# -########################################################################## -# Application Command line(no parameter needed) -# python inline_text_extraction_demo.py -########################################################################## - -""" -This python script provides an example of how to use OCI Document Understanding text extraction feature - -The configuration file used by service clients will be sourced from the default location (~/.oci/config) and the -CONFIG_PROFILE profile will be used. - -The sample attempts to extract text from an inline document -Successful run of this sample will create job results under object storage configured under output_location variable -""" -import oci -import uuid -import base64 - -# Setup basic variables -# Auth Config -CONFIG_PROFILE = "DEFAULT" -config = oci.config.from_file('~/.oci/config', CONFIG_PROFILE) - -# Compartment where processor job will be created -COMPARTMENT_ID = "" # e.g. "ocid1.compartment.oc1..aaaaaaaae5j73axsja5fnahbn23ilop3ynjkcg77mcvgryddz4pkh2t5ppaq"; - -#sample document -text_extraction_sample_string = None -with open("resources/text_extraction_demo.pdf", "rb") as document_file: - text_extraction_sample_string = base64.b64encode(document_file.read()).decode('utf-8') - -def create_processor_job_callback(times_called, response): - print("Waiting for processor lifecycle state to go into succeeded state:", response.data) - -aiservicedocument_client = oci.ai_document.AIServiceDocumentClientCompositeOperations(oci.ai_document.AIServiceDocumentClient(config=config)) - -# Text extraction feature -text_extraction_feature = oci.ai_document.models.DocumentTextExtractionFeature() - -# Setup the output location where processor job results will be created -output_location = oci.ai_document.models.OutputLocation() -output_location.namespace_name = "" # e.g. "axk2tfhlrens" -output_location.bucket_name = "" # e.g "output" -output_location.prefix = "" # e.g "demo" - -# Create a processor_job for text_extraction feature -create_processor_job_details_text_extraction = oci.ai_document.models.CreateProcessorJobDetails( - display_name=str(uuid.uuid4()), - compartment_id=COMPARTMENT_ID, - input_location=oci.ai_document.models.InlineDocumentContent(data=text_extraction_sample_string), - output_location=output_location, - processor_config=oci.ai_document.models.GeneralProcessorConfig(features=[text_extraction_feature])) - -print("Calling create_processor with create_processor_job_details_text_extraction:", create_processor_job_details_text_extraction) -create_processor_response = aiservicedocument_client.create_processor_job_and_wait_for_state( - create_processor_job_details=create_processor_job_details_text_extraction, - wait_for_states=[oci.ai_document.models.ProcessorJob.LIFECYCLE_STATE_SUCCEEDED], - waiter_kwargs={"wait_callback": create_processor_job_callback}) - -print("processor call succeeded with status: {} and request_id: {}.".format(create_processor_response.status, create_processor_response.request_id)) -processor_job: oci.ai_document.models.ProcessorJob = create_processor_response.data -print("create_processor_job_details_text_detection response: ", create_processor_response.data) - -print("Getting defaultObject.json from the output_location") -object_storage_client = oci.object_storage.ObjectStorageClient(config=config) -get_object_response = object_storage_client.get_object(namespace_name=output_location.namespace_name, - bucket_name=output_location.bucket_name, - object_name="{}/{}/_/results/defaultObject.json".format( - output_location.prefix, processor_job.id)) -print(str(get_object_response.data.content.decode())) \ No newline at end of file diff --git a/ai_services/document_understanding/python/text_extraction/object_storage_text_extraction_demo.py b/ai_services/document_understanding/python/text_extraction/object_storage_text_extraction_demo.py deleted file mode 100644 index eec323ab..00000000 --- a/ai_services/document_understanding/python/text_extraction/object_storage_text_extraction_demo.py +++ /dev/null @@ -1,87 +0,0 @@ -# coding: utf-8 -# Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved. -# This software is dual-licensed to you under the Universal Permissive License (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl or Apache License 2.0 as shown at http://www.apache.org/licenses/LICENSE-2.0. You may choose either license. - -########################################################################## -# object_storage_text_extraction_demo.py -# -# Supports Python 3 -########################################################################## -# Info: -# Object Storage Text Extraction Processor Job creation using OCI AI Document Understanding service. -# -########################################################################## -# Application Command line(no parameter needed) -# python object_storage_text_extraction_demo.py -########################################################################## - -""" -This python script provides an example of how to use OCI Document Understanding Service text extraction feature - -The configuration file used by service clients will be sourced from the default location (~/.oci/config) and the -CONFIG_PROFILE profile will be used. - -The sample attempts to extract text from a document in object storage -Successful run of this sample will create job results under object storage configured under output_location variable -""" -import oci -import uuid -import base64 - -# Setup basic variables -# Auth Config -CONFIG_PROFILE = "DEFAULT" -config = oci.config.from_file('~/.oci/config', CONFIG_PROFILE) - -# Compartment where processor job will be created -COMPARTMENT_ID = "" # e.g. "ocid1.compartment.oc1..aaaaaaaae5j73axsja5fnahbn23ilop3ynjkcg77mcvgryddz4pkh2t5ppaq"; - -def create_processor_job_callback(times_called, response): - print("Waiting for processor lifecycle state to go into succeeded state:", response.data) - -aiservicedocument_client = oci.ai_document.AIServiceDocumentClientCompositeOperations(oci.ai_document.AIServiceDocumentClient(config=config)) - -# Text extraction feature -text_extraction_feature = oci.ai_document.models.DocumentTextExtractionFeature() - -# Setup input location where document being processed is stored. -object_location = oci.ai_document.models.ObjectLocation() -object_location.namespace_name = "" # e.g. "axhh9gizbq5x" -object_location.bucket_name = "" # e.g "demo_examples" -object_location.object_name = "" # e.g "text_detection_extraction_demo.jpg" - -# Setup the output location where processor job results will be created -output_location = oci.ai_document.models.OutputLocation() -output_location.namespace_name = "" # e.g. "axk2tfhlrens" -output_location.bucket_name = "" # e.g "output" -output_location.prefix = "" # e.g "demo" - -# Create a processor_job for text_extraction feature -create_processor_job_details_text_extraction = oci.ai_document.models.CreateProcessorJobDetails( - display_name=str(uuid.uuid4()), - compartment_id=COMPARTMENT_ID, - input_location=oci.ai_document.models.ObjectStorageLocations(object_locations=[object_location]), - output_location=output_location, - processor_config=oci.ai_document.models.GeneralProcessorConfig(features=[text_extraction_feature])) - -print("Calling create_processor with create_processor_job_details_text_extraction:", create_processor_job_details_text_extraction) -create_processor_response = aiservicedocument_client.create_processor_job_and_wait_for_state( - create_processor_job_details=create_processor_job_details_text_extraction, - wait_for_states=[oci.ai_document.models.ProcessorJob.LIFECYCLE_STATE_SUCCEEDED], - waiter_kwargs={"wait_callback": create_processor_job_callback}) - -print("processor call succeeded with status: {} and request_id: {}.".format(create_processor_response.status, create_processor_response.request_id)) -processor_job: oci.ai_document.models.ProcessorJob = create_processor_response.data -print("create_processor_job_details_text_extraction response: ", create_processor_response.data) - -print("Getting result json from the output_location") -object_storage_client = oci.object_storage.ObjectStorageClient(config=config) -get_object_response = object_storage_client.get_object(namespace_name=output_location.namespace_name, - bucket_name=output_location.bucket_name, - object_name="{}/{}/{}_{}/results/{}.json".format( - output_location.prefix, processor_job.id, - object_location.namespace_name, - object_location.bucket_name, - object_location.object_name)) - -print(str(get_object_response.data.content.decode())) \ No newline at end of file