diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 12b192cf3b1b..03a06f09499b 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -32,7 +32,7 @@ Build Environment ================= For instructions on setting up your development environment, please see the -`BUILD.rst `__ file. +`BUILD.rst `__ file. Contribution Guidelines ======================= diff --git a/DEVELOPERS.rst b/DEVELOPERS.rst index 26a01582216a..2b6f813faa96 100644 --- a/DEVELOPERS.rst +++ b/DEVELOPERS.rst @@ -54,7 +54,7 @@ Build and Run CDAP Sandbox Locally with Maven The following builds and runs the CDAP sandbox via mvn package. For more details on development environments and build options, please see the -`BUILD.rst `__ file. +`BUILD.rst `__ file. - Obtain a fresh copy of the CDAP (GitHub) repo:: diff --git a/README.rst b/README.rst index 20614a1a2478..57e4699878e1 100644 --- a/README.rst +++ b/README.rst @@ -67,7 +67,7 @@ Build You can get started with CDAP by building directly from the latest source code:: - git clone https://github.com/caskdata/cdap.git + git clone https://github.com/cdapio/cdap.git cd cdap mvn clean package @@ -106,10 +106,10 @@ For quick guide to getting your system setup to contribute to CDAP, take a look Filing Issues: Bug Reports & Feature Requests --------------------------------------------- -Bugs and suggestions should be made by `filing an issue `__. +Bugs and suggestions should be made by `filing an issue `__. Existing issues can be browsed at `the CDAP project issues -`__. +`__. Pull Requests ------------- diff --git a/cdap-api/src/main/java/io/cdap/cdap/api/dataset/lib/PartitionedFileSetArguments.java b/cdap-api/src/main/java/io/cdap/cdap/api/dataset/lib/PartitionedFileSetArguments.java index d82495c0f912..e50ce5f5c1d0 100644 --- a/cdap-api/src/main/java/io/cdap/cdap/api/dataset/lib/PartitionedFileSetArguments.java +++ b/cdap-api/src/main/java/io/cdap/cdap/api/dataset/lib/PartitionedFileSetArguments.java @@ -128,7 +128,7 @@ public static Map getOutputPartitionMetadata(Map public static void setInputPartitionFilter(Map arguments, PartitionFilter filter) { // Serialize a singleton list for now. Support for multiple PartitionFilters can be added in the future. - // See: https://issues.cask.co/browse/CDAP-5618 + // See: https://cdap.atlassian.net/browse/CDAP-5618 arguments.put(INPUT_PARTITION_FILTER, GSON.toJson(Collections.singletonList(filter))); } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/batch/MapReduceRuntimeService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/batch/MapReduceRuntimeService.java index c858bbf08fb4..694d78934d06 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/batch/MapReduceRuntimeService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/batch/MapReduceRuntimeService.java @@ -222,7 +222,7 @@ protected void startUp() throws Exception { Map localizedUserResources = localizeUserResources(job, tempDir); // Override user-defined job name, since we set it and depend on the name. - // https://issues.cask.co/browse/CDAP-2441 + // https://cdap.atlassian.net/browse/CDAP-2441 String jobName = job.getJobName(); if (!jobName.isEmpty()) { LOG.warn("Job name {} is being overridden.", jobName); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/schedule/store/ProgramScheduleStoreDataset.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/schedule/store/ProgramScheduleStoreDataset.java index 06fea4c9d8ae..803561b87a8a 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/schedule/store/ProgramScheduleStoreDataset.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/schedule/store/ProgramScheduleStoreDataset.java @@ -275,7 +275,7 @@ public void deleteSchedules(Iterable scheduleIds, @Nullabl * @param appId the application id for which to delete the schedules * @return the IDs of the schedules that were deleted */ - // TODO: fix the bug that this method will return fake schedule id https://issues.cask.co/browse/CDAP-13626 + // TODO: fix the bug that this method will return fake schedule id https://cdap.atlassian.net/browse/CDAP-13626 public List deleteSchedules(ApplicationId appId, long deleteTime) throws IOException { List deleted = new ArrayList<>(); Collection> scanKeys = getScheduleKeysForApplicationScan(appId); @@ -301,7 +301,7 @@ public List deleteSchedules(ApplicationId appId, long deleteTime) th * @param programId the program id for which to delete the schedules * @return the IDs of the schedules that were deleted */ - // TODO: fix the bug that this method will return fake schedule id https://issues.cask.co/browse/CDAP-13626 + // TODO: fix the bug that this method will return fake schedule id https://cdap.atlassian.net/browse/CDAP-13626 public List deleteSchedules(ProgramId programId, long deleteTime) throws IOException { List deleted = new ArrayList<>(); Collection> scanKeys = getScheduleKeysForApplicationScan(programId.getParent()); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/ApplicationLifecycleService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/ApplicationLifecycleService.java index 59b5bca607c8..2ca62459c5e8 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/ApplicationLifecycleService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/ApplicationLifecycleService.java @@ -1395,7 +1395,7 @@ private void deleteAppMetadata(ApplicationId appId, ApplicationSpecification app // Remove metadata for the programs of the Application // TODO: Need to remove this we support prefix search of metadata type. - // See https://issues.cask.co/browse/CDAP-3669 + // See https://cdap.atlassian.net/browse/CDAP-3669 for (ProgramId programId : getAllPrograms(appId, appSpec)) { metadataServiceClient.drop(new MetadataMutation.Drop(programId.toMetadataEntity())); } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/AppMetadataStore.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/AppMetadataStore.java index c8d5e1def038..f87f2b61cfac 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/AppMetadataStore.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/store/AppMetadataStore.java @@ -1942,7 +1942,7 @@ public Map getRuns(ApplicationId applicationId, // TODO: getRun is duplicated in cdap-watchdog AppMetadataStore class. // Any changes made here will have to be made over there too. - // JIRA https://issues.cask.co/browse/CDAP-2172 + // JIRA https://cdap.atlassian.net/browse/CDAP-2172 @Nullable public RunRecordDetail getRun(ProgramRunId programRun) throws IOException { // Query active run record first diff --git a/cdap-app-fabric/src/main/java/org/apache/hadoop/fs/ftp/FTPInputStream.java b/cdap-app-fabric/src/main/java/org/apache/hadoop/fs/ftp/FTPInputStream.java index 854ed532564f..8f086b4112a0 100644 --- a/cdap-app-fabric/src/main/java/org/apache/hadoop/fs/ftp/FTPInputStream.java +++ b/cdap-app-fabric/src/main/java/org/apache/hadoop/fs/ftp/FTPInputStream.java @@ -30,7 +30,7 @@ * seeks are attempted to the current position. Position equality check logic in {@link * FTPInputStream#seek} is the only change from the original class in Hadoop. This change is * required since {@link LineRecordReader} calls {@link FTPInputStream#seek} with value of 0. TODO: - * This file can be removed once https://issues.cask.co/browse/CDAP-5387 is addressed. + * This file can be removed once https://cdap.atlassian.net/browse/CDAP-5387 is addressed. */ @InterfaceAudience.Private @InterfaceStability.Unstable diff --git a/cdap-app-fabric/src/main/java/org/apache/hadoop/fs/sftp/SFTPInputStream.java b/cdap-app-fabric/src/main/java/org/apache/hadoop/fs/sftp/SFTPInputStream.java index 1356bbe7dbff..65a6b2b42608 100644 --- a/cdap-app-fabric/src/main/java/org/apache/hadoop/fs/sftp/SFTPInputStream.java +++ b/cdap-app-fabric/src/main/java/org/apache/hadoop/fs/sftp/SFTPInputStream.java @@ -31,7 +31,7 @@ * seeks are attempted to the current position. Position equality check logic in {@link * SFTPInputStream#seek} is the only change from the original class in Hadoop. This change is * required since {@link LineRecordReader} calls {@link SFTPInputStream#seek} with value of 0. TODO: - * This file can be removed once https://issues.cask.co/browse/CDAP-5387 is addressed. + * This file can be removed once https://cdap.atlassian.net/browse/CDAP-5387 is addressed. */ class SFTPInputStream extends FSInputStream { diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/AppFabricTestBase.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/AppFabricTestBase.java index 4807c5599f84..21a983c1d324 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/AppFabricTestBase.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/AppFabricTestBase.java @@ -511,7 +511,7 @@ private HttpResponse addArtifact(Id.Artifact artifactId, InputSupplier props) { return !"true".equalsIgnoreCase(props.get(Constants.Dataset.TABLE_TX_DISABLED)); diff --git a/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/dataset2/lib/table/BufferingTable.java b/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/dataset2/lib/table/BufferingTable.java index ed5280c3aaa1..99b7f159c39c 100644 --- a/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/dataset2/lib/table/BufferingTable.java +++ b/cdap-data-fabric/src/main/java/io/cdap/cdap/data2/dataset2/lib/table/BufferingTable.java @@ -659,7 +659,7 @@ public void increment(byte[] row, byte[][] columns, long[] amounts) { @Override public boolean compareAndSwap(byte[] row, byte[] column, byte[] expectedValue, byte[] newValue) { ensureTransactionIsStarted(); - // TODO: add support for empty values; see https://issues.cask.co/browse/TEPHRA-45 for details. + // TODO: add support for empty values; see https://cdap.atlassian.net/browse/TEPHRA-45 for details. if (newValue != null && newValue.length == 0) { warnAboutEmptyValue(column); } diff --git a/cdap-distributions/src/emr/install.sh b/cdap-distributions/src/emr/install.sh index 49aa4e799c4e..1cda51e73863 100644 --- a/cdap-distributions/src/emr/install.sh +++ b/cdap-distributions/src/emr/install.sh @@ -106,7 +106,7 @@ curl -L -o ${__tmpdir}/install.sh https://www.chef.io/chef/install.sh && sudo ba # Clone CDAP repo echo "INFO: Checking out CDAP_BRANCH ${CDAP_BRANCH}" -git clone --depth 1 --branch ${CDAP_BRANCH} https://github.com/caskdata/cdap.git ${__gitdir} +git clone --depth 1 --branch ${CDAP_BRANCH} https://github.com/cdapio/cdap.git ${__gitdir} # Check out to specific tag if specified if [ -n "${CDAP_TAG}" ]; then diff --git a/cdap-distributions/src/hdinsight/pkg/install.sh b/cdap-distributions/src/hdinsight/pkg/install.sh index 8c5c8d3c26d2..fb5096eccfbb 100755 --- a/cdap-distributions/src/hdinsight/pkg/install.sh +++ b/cdap-distributions/src/hdinsight/pkg/install.sh @@ -57,7 +57,7 @@ __create_tmpdir curl -L -o ${__tmpdir}/install.sh https://www.chef.io/chef/install.sh && sudo bash ${__tmpdir}/install.sh -v ${CHEF_VERSION} || die "Failed to install chef" # Clone CDAP repo -git clone --depth 1 --branch ${CDAP_BRANCH} https://github.com/caskdata/cdap.git ${__gitdir} +git clone --depth 1 --branch ${CDAP_BRANCH} https://github.com/cdapio/cdap.git ${__gitdir} # Check out to specific tag if specified if [ -n "${CDAP_TAG}" ]; then diff --git a/cdap-distributions/src/packer/scripts/cookbook-setup.sh b/cdap-distributions/src/packer/scripts/cookbook-setup.sh index 95f7a490442e..04ba5d498248 100755 --- a/cdap-distributions/src/packer/scripts/cookbook-setup.sh +++ b/cdap-distributions/src/packer/scripts/cookbook-setup.sh @@ -23,7 +23,7 @@ die() { echo $*; exit 1; } export GIT_MERGE_AUTOEDIT=no # Grab cookbooks using knife -# Due to https://issues.cask.co/browse/CDAP-13308, we can no longer use knife cookbook site install +# Due to https://cdap.atlassian.net/browse/CDAP-13308, we can no longer use knife cookbook site install # for cb in cdap idea maven openssh; do # knife cookbook site install $cb || die "Cannot fetch cookbook $cb" # done diff --git a/cdap-docs/_common/_source/404.rst b/cdap-docs/_common/_source/404.rst index fde830fb7206..30bc7df9f1b6 100644 --- a/cdap-docs/_common/_source/404.rst +++ b/cdap-docs/_common/_source/404.rst @@ -75,5 +75,5 @@ Post a message at one of the two CDAP groups: - `CDAP Developers: cdap-dev@googlegroups.com `__ -We also have a `JIRA for filing issues. `__ +We also have a `JIRA for filing issues. `__ diff --git a/cdap-docs/_common/common_conf.py b/cdap-docs/_common/common_conf.py index a2f567cc2127..18309186ae37 100644 --- a/cdap-docs/_common/common_conf.py +++ b/cdap-docs/_common/common_conf.py @@ -261,7 +261,7 @@ def get_git_build_vars(): 'cdap-studio-artifact': ("%s/pipelines/studio?artifactType=%%s" % _cdap_ui_base, None), 'cdap-java-source-github': (cdap_java_source_github_pattern , None), 'cdap-security-extn-source-github': (cdap_security_extn_github_pattern, None), - 'cask-issue': ('https://issues.cask.co/browse/%s', ''), + 'cask-issue': ('https://cdap.atlassian.net/browse/%s', ''), 'cask-repository-parcels-cdap': ("http://repository.cask.co/parcels/cdap/%s/%%s" % short_version, None), 'cdap-guides': (cdap_guides_github_pattern, None), 'spark-docs': ("https://spark.apache.org/docs/%s/%%s" % spark_version, None), diff --git a/cdap-docs/admin-manual/source/_includes/installation/configuration.txt b/cdap-docs/admin-manual/source/_includes/installation/configuration.txt index 99d1556080c1..a276a1ca0dfe 100644 --- a/cdap-docs/admin-manual/source/_includes/installation/configuration.txt +++ b/cdap-docs/admin-manual/source/_includes/installation/configuration.txt @@ -169,7 +169,7 @@ for CDAP to run successfully. .. _|distribution|-configuration-explore-service: **Note:** Some versions of Hive contain a bug that may prevent the CDAP Explore Service from starting - up. See `CDAP-1865 `__ for more information about the issue. + up. See `CDAP-1865 `__ for more information about the issue. If the CDAP Explore Service fails to start and you see a ``javax.jdo.JDODataStoreException: Communications link failure`` in the log, try adding this property to the Hive ``hive-site.xml`` file:: diff --git a/cdap-docs/admin-manual/source/installation/ambari.rst b/cdap-docs/admin-manual/source/installation/ambari.rst index ae68bc4d97c1..21599cecde04 100644 --- a/cdap-docs/admin-manual/source/installation/ambari.rst +++ b/cdap-docs/admin-manual/source/installation/ambari.rst @@ -24,8 +24,8 @@ Installation using Apache Ambari **not** with Ambari. - A number of features are currently planned to be added, including: - - select `CDAP metrics `__ and - - a full `smoke test of CDAP functionality `__ after installation. + - select `CDAP metrics `__ and + - a full `smoke test of CDAP functionality `__ after installation. - If you are installing CDAP with the intention of using *replication,* see these instructions on :ref:`CDAP Replication ` *before* installing or starting CDAP. diff --git a/cdap-docs/developer-manual/source/metadata/audit-logging.rst b/cdap-docs/developer-manual/source/metadata/audit-logging.rst index 5307c82b9947..bf69cd359254 100644 --- a/cdap-docs/developer-manual/source/metadata/audit-logging.rst +++ b/cdap-docs/developer-manual/source/metadata/audit-logging.rst @@ -21,7 +21,7 @@ streams), it includes access information used to generate the entity's :ref:`lin Please note that audit logs are not published during a CDAP upgrade, as CDAP services are not available. Hence, any application which uses CDAP audit logs to sync metadata will go out of sync with respect to changes made during the upgrade. Please see -`CDAP-5954 `__ for details. +`CDAP-5954 `__ for details. .. _audit-logging-supported-audit-events: diff --git a/cdap-docs/faqs/source/general.rst b/cdap-docs/faqs/source/general.rst index 9e087e16a757..b84fb6241da6 100644 --- a/cdap-docs/faqs/source/general.rst +++ b/cdap-docs/faqs/source/general.rst @@ -25,7 +25,7 @@ What platforms are supported by the CDAP Sandbox? ------------------------------------------------------- The CDAP Sandbox has been extensively tested on Mac OS X and Linux. CDAP on Windows has not been as extensively tested. If you have any issues with CDAP on Windows, help us by -`filing a ticket `__. +`filing a ticket `__. What programming languages are supported by CDAP? ------------------------------------------------- @@ -66,7 +66,7 @@ I've found a bug in CDAP. How do I file an issue? ------------------------------------------------- You can use the `CDAP User GoogleGroup `__ to report issues or file a ticket using the available `CDAP JIRA system -`__. +`__. What User Groups and Mailing Lists are available about CDAP? ------------------------------------------------------------ diff --git a/cdap-docs/faqs/source/index.rst b/cdap-docs/faqs/source/index.rst index 23f5289c6e6b..8bfc3e3614d8 100644 --- a/cdap-docs/faqs/source/index.rst +++ b/cdap-docs/faqs/source/index.rst @@ -25,7 +25,7 @@ Check our Issues Database for known issues ------------------------------------------ When trying to solve an issue, one source of information is the CDAP Issues database. The `unresolved issues can be browsed -`__; +`__; and using the search box in the upper-right, you can look for issues that contain a particular problem or keyword: .. image:: _images/faqs-cdap-issues.png diff --git a/cdap-docs/reference-manual/source/release-notes.rst b/cdap-docs/reference-manual/source/release-notes.rst index 9686dc748277..b8507913cdb5 100644 --- a/cdap-docs/reference-manual/source/release-notes.rst +++ b/cdap-docs/reference-manual/source/release-notes.rst @@ -4044,41 +4044,41 @@ Bug Fixes Bug Fixes --------- -- `CDAP-4388 `__ - Fixed a race +- `CDAP-4388 `__ - Fixed a race condition bug in ResourceCoordinator that prevented performing partition assignment in the correct order. It affects the metrics processor and stream coordinator. -- `CDAP-5855 `__ - Avoid the +- `CDAP-5855 `__ - Avoid the cancellation of delegation tokens upon completion of Explore-launched MapReduce and Spark jobs, as these delegation tokens are shared by CDAP system services. -- `CDAP-5868 `__ - Removed +- `CDAP-5868 `__ - Removed 'SNAPSHOT' from the artifact version of apps created by default by the CDAP UI. This fixes deploying Cask Tracker and Navigator apps, enabling Cask Tracker from the CDAP UI. -- `CDAP-5884 `__ - Fixed a bug +- `CDAP-5884 `__ - Fixed a bug that caused SDK builds to fail when using 3.3.x versions of maven. -- `CDAP-5887 `__ - Fixed the +- `CDAP-5887 `__ - Fixed the Hydrator upgrade tool to correctly write out pipeline configs that failed to upgrade. -- `CDAP-5889 `__ - The CDAP +- `CDAP-5889 `__ - The CDAP Standalone now deploys and starts the Cask Tracker app in the default namespace if the Tracker artifact is present. -- `CDAP-5898 `__ - Shutdown +- `CDAP-5898 `__ - Shutdown external processes started by CDAP (Zookeeper and Kafka) when there is an error during either startup or shutdown of CDAP. -- `CDAP-5907 `__ - Fixed an +- `CDAP-5907 `__ - Fixed an issue where parsing of an AVRO schema was failing when it included optional fields such as 'doc' or 'default'. -- `CDAP-5947 `__ - Fixed a bug +- `CDAP-5947 `__ - Fixed a bug in the BatchReadableRDD so that it won't skip records when used by DataFrame. @@ -4088,72 +4088,72 @@ Known Issues be lost and *WARN* log messages will be logged that tell about the inability to process old data in the old format. -- `CDAP-797 `__ - +- `CDAP-797 `__ - When running secure Hadoop clusters, debug logs from MapReduce programs are not available. -- `CDAP-1007 `__ - +- `CDAP-1007 `__ - If the Hive Metastore is restarted while the CDAP Explore Service is running, the Explore Service remains alive, but becomes unusable. To correct, restart the CDAP Master |---| which will restart all services |---| as described under "Starting CDAP Services" for your particular Hadoop distribution in the `Installation documentation `__. -- `CDAP-1587 `__ - +- `CDAP-1587 `__ - CDAP internally creates tables in the "user" space that begin with the word ``"system"``. User datasets with names starting with ``"system"`` can conflict if they were to match one of those names. To avoid this, do not start any datasets with the word ``"system"``. -- `CDAP-2632 `__ - +- `CDAP-2632 `__ - The application in the `cdap-kafka-ingest-guide `__ does not run on Ubuntu 14.x as of CDAP 3.0.x. -- `CDAP-2721 `__ - +- `CDAP-2721 `__ - Metrics for `FileSets `__ can show zero values even if there is data present, because FileSets do not emit metrics (`CDAP-587 - `__). + `__). -- `CDAP-2831 `__ - +- `CDAP-2831 `__ - A workflow that is scheduled by time will not be run between the failure of the primary master and the time that the secondary takes over. This scheduled run will not be triggered at all. -- `CDAP-2920 `__ - Spark jobs on a +- `CDAP-2920 `__ - Spark jobs on a Kerberos-enabled CDAP cluster cannot run longer than the delegation token expiration. -- `CDAP-2945 `__ - +- `CDAP-2945 `__ - If the input partition filter for a PartitionedFileSet does not match any partitions, MapReduce jobs can fail. -- `CDAP-3000 `__ - +- `CDAP-3000 `__ - The Workflow token is in an inconsistent state for nodes in a fork while the nodes of the fork are still running. It becomes consistent after the join. -- `CDAP-3221 `__ - +- `CDAP-3221 `__ - When running in Standalone CDAP, if a MapReduce job fails repeatedly, then the SDK hits an out-of-memory exception due to ``perm gen``. The Standalone needs restarting at this point. -- `CDAP-3262 `__ - +- `CDAP-3262 `__ - For Microsoft Windows, the Standalone CDAP scripts can fail when used with a JAVA_HOME that is defined as a path with spaces in it. A workaround is to use a definition of JAVA_HOME that does not include spaces, such as ``C:\PROGRA~1\Java\jdk1.7.0_79\bin`` or ``C:\ProgramData\Oracle\Java\javapath``. -- `CDAP-3492 `__ - +- `CDAP-3492 `__ - In the CDAP CLI, executing ``select *`` from a dataset with many fields generates an error. -- `CDAP-3641 `__ - +- `CDAP-3641 `__ - A RESTful API call to retrieve workflow statistics hangs if units (such as "s" for seconds) are not provided as part of the query. -- `CDAP-3750 `__ - +- `CDAP-3750 `__ - If a table schema contains a field name that is a reserved word in the Hive DDL, ``'enable explore'`` fails. -- `CDAP-5900 `__ - During the +- `CDAP-5900 `__ - During the upgrade to CDAP 3.4.1, publishing to Kafka is halted because the CDAP Kafka service is not running. As a consequence, any applications that sync to the CDAP metadata will become out-of-sync as changes to the @@ -4165,292 +4165,292 @@ Known Issues API Changes ----------- -- `CDAP-5082 `__ - Added a new Spark Java and Scala API. +- `CDAP-5082 `__ - Added a new Spark Java and Scala API. New Features ------------ -- `CDAP-20 `__ - Removed dependency on the Guava +- `CDAP-20 `__ - Removed dependency on the Guava library from the ``cdap-api`` module. Applications are now free to use a Guava library version of their choice. -- `CDAP-3051 `__ - Added capability for programs to +- `CDAP-3051 `__ - Added capability for programs to perform administrative dataset operations (create, update, truncate, drop). -- `CDAP-3854 `__ - Added the capability to +- `CDAP-3854 `__ - Added the capability to configure Kafka topic for logs and notifications using the ``cdap-site.xml``. -- `CDAP-3980 `__ - MapReduce programs submitted via CDAP +- `CDAP-3980 `__ - MapReduce programs submitted via CDAP now support multiple configured inputs. -- `CDAP-4807 `__ - Added an ODBC 3.0 Driver for +- `CDAP-4807 `__ - Added an ODBC 3.0 Driver for CDAP Datasets for Windows-based applications that support an ODBC interface. -- `CDAP-4970 `__ - Added capability to fetch the +- `CDAP-4970 `__ - Added capability to fetch the schema from a JDBC source specified for a Database plugin from inside Cask Hydrator. -- `CDAP-5011 `__ - Added a CDAP extension *Cask Tracker*: +- `CDAP-5011 `__ - Added a CDAP extension *Cask Tracker*: data discovery with metadata, audit, and lineage. -- `CDAP-5146 `__ - Added a new Cask Hydrator +- `CDAP-5146 `__ - Added a new Cask Hydrator ``batchaggregator`` plugin type. An aggregator operates on a collection of records, grouping them by a key and performing an aggregation on each group. -- `CDAP-5172 `__ - Added support for +- `CDAP-5172 `__ - Added support for authorization extensions in CDAP. Extensions extend an ``Authorizer`` class and provide a bundle jar containing all their required dependencies. This jar is then specified using the property ``security.authorization.extension.jar.path`` in the ``cdap-site.xml``. -- `CDAP-5191 `__ - Added an ``FTPBatchSource`` +- `CDAP-5191 `__ - Added an ``FTPBatchSource`` that can fetch data from an FTP server in a batch pipeline of Cask Hydrator. -- `CDAP-5205 `__ - Added a global search across +- `CDAP-5205 `__ - Added a global search across all CDAP entities in the CDAP UI. -- `CDAP-5274 `__ - The Cask Hydrator Studio now +- `CDAP-5274 `__ - The Cask Hydrator Studio now includes the capability to configure a new type of pipeline, a "data pipeline" (beta feature). -- `CDAP-5360 `__ - The CDAP UI now supports +- `CDAP-5360 `__ - The CDAP UI now supports ``Sparksink`` and ``Sparkcompute`` plugin types, included in a new "data pipeline" artifact. -- `CDAP-5361 `__ - Added a ``SparkTransform`` +- `CDAP-5361 `__ - Added a ``SparkTransform`` plugin type, which allows the running of a Spark job that operates as a transform in an ETL batch pipeline. -- `CDAP-5362 `__ - Added a ``SparkSink`` plugin +- `CDAP-5362 `__ - Added a ``SparkSink`` plugin type, which allows the running of a Spark job (such as machine learning) on the output of an ETL batch pipeline. -- `CDAP-5392 `__ - Added support for +- `CDAP-5392 `__ - Added support for ``FormatSpecification`` in Spark when consuming data from a stream. -- `CDAP-5446 `__ - Added an example application +- `CDAP-5446 `__ - Added an example application demonstrating the use of Spark Streaming with machine-learning and spam classifying. -- `CDAP-5504 `__ - Added experimental support for +- `CDAP-5504 `__ - Added experimental support for using Spark as an execution engine for CDAP Explore. -- `CDAP-5707 `__ - Added support for using Tez as +- `CDAP-5707 `__ - Added support for using Tez as an execution engine for CDAP Explore. -- `CDAP-5846 `__ - Bundled `Node.js +- `CDAP-5846 `__ - Bundled `Node.js `__ with the CDAP UI RPM and DEB packages and with the CDAP Parcels. Improvements ------------ -- `CDAP-4071 `__ - MapReduce programs can now be +- `CDAP-4071 `__ - MapReduce programs can now be configured to write metadata for each partition created using a ``DynamicPartitioner``. -- `CDAP-4117 `__ - Fixed an issue of not using +- `CDAP-4117 `__ - Fixed an issue of not using the correct user account to access HDFS when submitting a YARN application through Apache Twill, which caused a cleanup failure (and a confusing error message) upon application termination. -- `CDAP-4644 `__ - Workflow logs now contain logs +- `CDAP-4644 `__ - Workflow logs now contain logs from all of the actions executed by a workflow. -- `CDAP-4842 `__ - Added a ``hydrator-test`` +- `CDAP-4842 `__ - Added a ``hydrator-test`` module that contains mock plugins for unit testing Hydrator plugins. -- `CDAP-4925 `__ - Added to the CDAP test +- `CDAP-4925 `__ - Added to the CDAP test framework the ability to delete applications and artifacts, retrieve application information, update an application, and write and remove properties for artifacts. -- `CDAP-4955 `__ - Added a 'postaction' Cask +- `CDAP-4955 `__ - Added a 'postaction' Cask Hydrator plugin type that runs at the end of a pipeline run, irregardless of whether the run succeeded or failed. -- `CDAP-5001 `__ - Downloading an explore query +- `CDAP-5001 `__ - Downloading an explore query from the CDAP UI will now stream the results directly to the client. -- `CDAP-5037 `__ - Added a configuration property +- `CDAP-5037 `__ - Added a configuration property to Cask Hydrator TimePartitionedFileSet (TPFS) sinks that will clean out data that is older than a threshold amount of time. -- `CDAP-5039 `__ - Added runtime macros to +- `CDAP-5039 `__ - Added runtime macros to database and post-action Cask Hydrator plugins. -- `CDAP-5042 `__ - Added a ``numSplits`` +- `CDAP-5042 `__ - Added a ``numSplits`` configuration property to Cask Hydrator database sources to allow users to configure how many splits should be used for an import query. -- `CDAP-5046 `__ - The CDAP UI now allows a +- `CDAP-5046 `__ - The CDAP UI now allows a plugin developer to use a "textarea" in node configurations for displaying a plugin property. -- `CDAP-5075 `__ - Programs now have a +- `CDAP-5075 `__ - Programs now have a ``logical.start.time`` runtime argument that is populated by the system to be the start time of the program. The argument can be overridden just as other runtime arguments. -- `CDAP-5082 `__ - Added support for Spark +- `CDAP-5082 `__ - Added support for Spark streaming (to interact with the transactional datasets in CDAP), and support for concurrent Spark execution through Workflow forking. -- `CDAP-5178 `__ - Changed the format of the Cask +- `CDAP-5178 `__ - Changed the format of the Cask Hydrator configuration. All pipeline stages are now together in a "stages" array instead of being broken up into separate "source", "transforms", and "sinks" arrays. -- `CDAP-5181 `__ - Added an HTTP RESTful endpoint +- `CDAP-5181 `__ - Added an HTTP RESTful endpoint to retrieve the state of all nodes in a workflow. -- `CDAP-5182 `__ - Added an API to retrieve the +- `CDAP-5182 `__ - Added an API to retrieve the properties that were used to configure (or reconfigure) a dataset. -- `CDAP-5207 `__ - Removed dependency on Guava +- `CDAP-5207 `__ - Removed dependency on Guava from the ``cdap-proto`` module. -- `CDAP-5228 `__ - Added support for CDH 5.7. +- `CDAP-5228 `__ - Added support for CDH 5.7. -- `CDAP-5330 `__ - The stream creation endpoint +- `CDAP-5330 `__ - The stream creation endpoint now accepts a stream configuration (with TTL, description, format specification, and notification threshold). -- `CDAP-5376 `__ - Added an API for MapReduce to +- `CDAP-5376 `__ - Added an API for MapReduce to retrieve information about the enclosing workflow, including its run ID. -- `CDAP-5378 `__ - Added access to workflow +- `CDAP-5378 `__ - Added access to workflow information in a Spark program when it is executed inside a workflow. -- `CDAP-5424 `__ - Added the ability to track the +- `CDAP-5424 `__ - Added the ability to track the lineage of external sources and sinks in a Cask Hydrator pipeline. -- `CDAP-5512 `__ - Extended the workflow APIs to +- `CDAP-5512 `__ - Extended the workflow APIs to allow the use of plugins. -- `CDAP-5664 `__ - Introduced a ``referenceName`` +- `CDAP-5664 `__ - Introduced a ``referenceName`` property (used for lineage and annotation metadata) into all external sources and sinks. This needs to be set before using any of these plugins. -- `CDAP-5779 `__ - Upgraded the Tephra version in +- `CDAP-5779 `__ - Upgraded the Tephra version in CDAP to 0.7.1. Bug Fixes --------- -- `CDAP-3498 `__ - Upgraded CDAP to use +- `CDAP-3498 `__ - Upgraded CDAP to use Apache Twill ``0.7.0-incubating`` with numerous new features, improvements, and bug fixes. See the `Apache Twill release notes `__ for details. -- `CDAP-3584 `__ - Upon transaction rollback, a +- `CDAP-3584 `__ - Upon transaction rollback, a ``PartitionedFileSet`` now rolls back the files for the partitions that were added and/or removed in that transaction. -- `CDAP-3749 `__ - Fixed a bug with the database +- `CDAP-3749 `__ - Fixed a bug with the database plugins that required a password to be specified if the user was specified, even if the password was empty. -- `CDAP-4060 `__ - Added the status for custom +- `CDAP-4060 `__ - Added the status for custom actions in workflow diagrams. -- `CDAP-4143 `__ - Fixed a problem with the +- `CDAP-4143 `__ - Fixed a problem with the database source where a semicolon at the end of the query would cause an error. -- `CDAP-4692 `__ - The CDAP UI now prevents users +- `CDAP-4692 `__ - The CDAP UI now prevents users from accidentally losing their DAG by showing a browser-native popup for a confirmation before navigating away from the Cask Hydrator Studio view. -- `CDAP-4695 `__ - Fixed an issue in the Windows +- `CDAP-4695 `__ - Fixed an issue in the Windows CDAP SDK where streams could not be deleted. -- `CDAP-4735 `__ - Fixed an issue that made Java +- `CDAP-4735 `__ - Fixed an issue that made Java extensions unavailable to programs, fixing the JavaScript-based Hydrator transforms under Java 8. -- `CDAP-4908 `__ - Removed ``tableName`` as a +- `CDAP-4908 `__ - Removed ``tableName`` as a required setting from database sources, since the ``importQuery`` is sufficient. -- `CDAP-4921 `__ - Renamed the Hydrator +- `CDAP-4921 `__ - Renamed the Hydrator ``Teradata`` batch source to ``Database``. The previous ``Database`` source is no longer supported. -- `CDAP-4982 `__ - Changed the Cask Hydrator +- `CDAP-4982 `__ - Changed the Cask Hydrator LogParser transform ``logFormat`` field from a textbox to a dropdown. -- `CDAP-5041 `__ - Changed several +- `CDAP-5041 `__ - Changed several ``ExploreConnection`` methods to be no-ops instead of throwing exceptions. -- `CDAP-5062 `__ - Added a ``fetch.size`` +- `CDAP-5062 `__ - Added a ``fetch.size`` connection setting to the JDBC driver to control the number of rows fetched per database cursor, and increased the default fetch size from 50 to 1000. -- `CDAP-5092 `__ - Fixed a problem that prevented +- `CDAP-5092 `__ - Fixed a problem that prevented applications written in Scala from being deployed. -- `CDAP-5103 `__ - Fixed a problem so that when the +- `CDAP-5103 `__ - Fixed a problem so that when the schema for a view was not explicitly specified, the view system metadata will include the default schema for the specified format if that is available. -- `CDAP-5131 `__ - Fixed a problem when filtering +- `CDAP-5131 `__ - Fixed a problem when filtering plugins by their extension plugin type; filtering by the extensions plugin type was returning extra results for any plugins that did not have an extension. -- `CDAP-5177 `__ - Fixed a problem with +- `CDAP-5177 `__ - Fixed a problem with PartitionConsumer not appropriately handling partitions that had been deleted since they were added to the working set. -- `CDAP-5241 `__ - Fixed a problem with metadata +- `CDAP-5241 `__ - Fixed a problem with metadata for a dataset not being deleted when a dataset was deleted. -- `CDAP-5267 `__ - Fixed a problem with the +- `CDAP-5267 `__ - Fixed a problem with the ``PartitionFilter.ALWAYS_MATCH`` not working as an input partition filter. ``PartitionFilter`` is now serialized into one key of the runtime arguments, to support serialization of ``PartitionFilter.ALWAYS_MATCH``. If there are additional fields in the ``PartitionFilter`` that do not exist in the partitioning, the filter will then never match. -- `CDAP-5272 `__ - Fixed a problem with a null +- `CDAP-5272 `__ - Fixed a problem with a null pointer exception when null values were written to a database sink in Cask Hydrator. -- `CDAP-5280 `__ - Corrected the documentation of +- `CDAP-5280 `__ - Corrected the documentation of the Query HTTP RESTful API for the retrieving of the status of a query. -- `CDAP-5297 `__ - Fixed a problem with the CDAP +- `CDAP-5297 `__ - Fixed a problem with the CDAP UI not supporting pipelines created using previous versions of Cask Hydrator. The UI now shows appropriate information to upgrade the pipeline to be able to view it in the UI. -- `CDAP-5417 `__ - Fixed an issue with running +- `CDAP-5417 `__ - Fixed an issue with running the CDAP examples in the CDAP SDK under Windows by setting appropriate memory requirements in the ``cdap.bat`` start script. -- `CDAP-5460 `__ - Fixed a problem with the +- `CDAP-5460 `__ - Fixed a problem with the workflow Spark programs status not being updated in the CDAP UI on the program list screen when it is run as a part of Workflow. -- `CDAP-5463 `__ - Fixed an issue when changing +- `CDAP-5463 `__ - Fixed an issue when changing the number of instances of a worker or service. -- `CDAP-5513 `__ - Fixed a problem with the +- `CDAP-5513 `__ - Fixed a problem with the update of metadata indexes so that search results reflect metadata updates correctly. -- `CDAP-5550 `__ - Fixed a problem with the +- `CDAP-5550 `__ - Fixed a problem with the workflow statistics HTTP RESTful endpoint. The endpoint now has a default limit of 10 and a default interval of 10 seconds. -- `CDAP-5557 `__ - Fixed a problem of not showing +- `CDAP-5557 `__ - Fixed a problem of not showing an appropriate error message in the node configuration when the CDAP backend returns 404 for a plugin property. -- `CDAP-5583 `__ - Added the ability to support +- `CDAP-5583 `__ - Added the ability to support multiple sources in the CDAP UI while constructing a pipeline. -- `CDAP-5619 `__ - Fixed a problem with the +- `CDAP-5619 `__ - Fixed a problem with the import of a pipeline configuration. If the imported pipeline config doesn't have artifact information for a plugin, the CDAP UI now defaults to the latest artifact from the list of artifacts sent by the backend. -- `CDAP-5629 `__ - Fixed a problem with losing +- `CDAP-5629 `__ - Fixed a problem with losing metadata after changing the stream format on a MapR cluster by avoiding the use of Hive keywords in the CLF format field names; the 'date' field was renamed to 'request_time'. -- `CDAP-5634 `__ - Fixed a performance issue when +- `CDAP-5634 `__ - Fixed a performance issue when rendering/scrolling through large input or output schemas for a plugin in the CDAP UI. -- `CDAP-5652 `__ - Added command line interface +- `CDAP-5652 `__ - Added command line interface command to retrieve the workflow node states. -- `CDAP-5793 `__ - CDAP Explore jobs properly use +- `CDAP-5793 `__ - CDAP Explore jobs properly use the latest/updated delegation tokens. -- `CDAP-5844 `__ - Fixed a problem with the +- `CDAP-5844 `__ - Fixed a problem with the updating of the HDFS delegation token for HA mode. Deprecated and Removed Features @@ -4475,70 +4475,70 @@ Known Issues be lost and *WARN* log messages will be logged that tell about the inability to process old data in the old format. -- `CDAP-797 `__ - +- `CDAP-797 `__ - When running secure Hadoop clusters, debug logs from MapReduce programs are not available. -- `CDAP-1007 `__ - +- `CDAP-1007 `__ - If the Hive Metastore is restarted while the CDAP Explore Service is running, the Explore Service remains alive, but becomes unusable. To correct, restart the CDAP Master |---| which will restart all services |---| as described under "Starting CDAP Services" for your particular Hadoop distribution in the `Installation documentation `__. -- `CDAP-1587 `__ - +- `CDAP-1587 `__ - CDAP internally creates tables in the "user" space that begin with the word ``"system"``. User datasets with names starting with ``"system"`` can conflict if they were to match one of those names. To avoid this, do not start any datasets with the word ``"system"``. -- `CDAP-2632 `__ - +- `CDAP-2632 `__ - The application in the `cdap-kafka-ingest-guide `__ does not run on Ubuntu 14.x as of CDAP 3.0.x. -- `CDAP-2721 `__ - +- `CDAP-2721 `__ - Metrics for `FileSets `__ can show zero values even if there is data present, because FileSets do not emit metrics (`CDAP-587 - `__). + `__). -- `CDAP-2831 `__ - +- `CDAP-2831 `__ - A workflow that is scheduled by time will not be run between the failure of the primary master and the time that the secondary takes over. This scheduled run will not be triggered at all. -- `CDAP-2920 `__ - Spark jobs on a +- `CDAP-2920 `__ - Spark jobs on a Kerberos-enabled CDAP cluster cannot run longer than the delegation token expiration. -- `CDAP-2945 `__ - +- `CDAP-2945 `__ - If the input partition filter for a PartitionedFileSet does not match any partitions, MapReduce jobs can fail. -- `CDAP-3000 `__ - +- `CDAP-3000 `__ - The Workflow token is in an inconsistent state for nodes in a fork while the nodes of the fork are still running. It becomes consistent after the join. -- `CDAP-3221 `__ - +- `CDAP-3221 `__ - When running in Standalone CDAP, if a MapReduce job fails repeatedly, then the SDK hits an out-of-memory exception due to ``perm gen``. The Standalone needs restarting at this point. -- `CDAP-3262 `__ - +- `CDAP-3262 `__ - For Microsoft Windows, the Standalone CDAP scripts can fail when used with a JAVA_HOME that is defined as a path with spaces in it. A workaround is to use a definition of JAVA_HOME that does not include spaces, such as ``C:\PROGRA~1\Java\jdk1.7.0_79\bin`` or ``C:\ProgramData\Oracle\Java\javapath``. -- `CDAP-3492 `__ - +- `CDAP-3492 `__ - In the CDAP CLI, executing ``select *`` from a dataset with many fields generates an error. -- `CDAP-3641 `__ - +- `CDAP-3641 `__ - A RESTful API call to retrieve workflow statistics hangs if units (such as "s" for seconds) are not provided as part of the query. -- `CDAP-3750 `__ - +- `CDAP-3750 `__ - If a table schema contains a field name that is a reserved word in the Hive DDL, ``'enable explore'`` fails. @@ -4548,7 +4548,7 @@ Known Issues Bug Fix ------- -- `CDAP-5350 `__ - Fixed an issue that prevented +- `CDAP-5350 `__ - Fixed an issue that prevented MapReduce programs from running on clusters with encryption. @@ -4557,35 +4557,35 @@ Bug Fix Improvements ------------ -- `CDAP-5047 `__ - Added a `Batch Source Plugin +- `CDAP-5047 `__ - Added a `Batch Source Plugin `__ to read from Microsoft Azure Blob Storage. -- `CDAP-5134 `__ - Added support for CDH 5.6 to CDAP. +- `CDAP-5134 `__ - Added support for CDH 5.6 to CDAP. Bug Fixes --------- -- `CDAP-4967 `__ - Fixed a schema-parsing bug +- `CDAP-4967 `__ - Fixed a schema-parsing bug that prevented the use of schemas where a record is used both as a top-level field and also used inside a different record field. -- `CDAP-5019 `__ - Worked around two issues +- `CDAP-5019 `__ - Worked around two issues (`SPARK-13441 `__ and `YARN-4727 `__) that prevented launching Spark jobs on CDH (Cloudera Data Hub) clusters managed with Cloudera Manager when using Spark 1.4 or greater. -- `CDAP-5063 `__ - Fixed a problem with +- `CDAP-5063 `__ - Fixed a problem with the CDAP Master not starting when CDAP and the HiveServer2 services are running on the same node in an Ambari cluster. -- `CDAP-5076 `__ - Fixed a problem with the CDAP +- `CDAP-5076 `__ - Fixed a problem with the CDAP CLI command "update app" that was parsing the application config incorrectly. -- `CDAP-5094 `__ - Fixed a problem where the explore +- `CDAP-5094 `__ - Fixed a problem where the explore schema fileset property was being ignored unless an explore format was also present. -- `CDAP-5137 `__ - Fix a problem with Spark jobs +- `CDAP-5137 `__ - Fix a problem with Spark jobs not being submitted to the appropriate YARN scheduler queue set for the namespace. @@ -4594,141 +4594,141 @@ Bug Fixes Improvements ------------ -- `CDAP-4602 `__ - Updated CDAP to use +- `CDAP-4602 `__ - Updated CDAP to use Tephra 0.6.5. -- `CDAP-4708 `__ - Added system metadata to +- `CDAP-4708 `__ - Added system metadata to existing entities. -- `CDAP-4723 `__ - Improved the Hydrator plugin +- `CDAP-4723 `__ - Improved the Hydrator plugin archetypes to include build steps to build the deployment JSON for the artifact. -- `CDAP-4773 `__ - Improved the error logging for +- `CDAP-4773 `__ - Improved the error logging for the Master Stream service when it can't connect to the CDAP AppFabric server. Bug Fixes --------- -- `CDAP-4117 `__ - Fixed an issue of not using +- `CDAP-4117 `__ - Fixed an issue of not using the correct user to access HDFS when submitting a YARN application through Apache Twill, which caused cleanup failure on application termination. -- `CDAP-4613 `__ - Fixed a problem with tooltips +- `CDAP-4613 `__ - Fixed a problem with tooltips not appearing in Flow and Workflow diagrams displayed in the Firefox browser. -- `CDAP-4679 `__ - The Hydrator UI now prevents +- `CDAP-4679 `__ - The Hydrator UI now prevents drafts from being created with a name of an already-existing draft. This prevents overwriting of existing drafts. -- `CDAP-4688 `__ - Improved the metadata search +- `CDAP-4688 `__ - Improved the metadata search to return matching entities from both the specified namespace and the system namespace. -- `CDAP-4689 `__ - Fixed a problem when using an +- `CDAP-4689 `__ - Fixed a problem when using an Hbase sink as one of multiple sinks in a Hydrator pipeline. -- `CDAP-4720 `__ - Fixed an issue where system +- `CDAP-4720 `__ - Fixed an issue where system metadata updates were not being published to Kafka. -- `CDAP-4721 `__ - Fixed an issue where metadata +- `CDAP-4721 `__ - Fixed an issue where metadata updates wouldn't be sent when certain entities were deleted. -- `CDAP-4740 `__ - Added validation to the JSON +- `CDAP-4740 `__ - Added validation to the JSON imported in the Hydrator UI. -- `CDAP-4741 `__ - Fixed a bug with deleting +- `CDAP-4741 `__ - Fixed a bug with deleting artifact metadata when an artifact was deleted. -- `CDAP-4743 `__ - Fixed the Node.js server proxy +- `CDAP-4743 `__ - Fixed the Node.js server proxy to handle all backend errors (with and without statusCodes). -- `CDAP-4745 `__ - Fixed a bug in the Hydrator +- `CDAP-4745 `__ - Fixed a bug in the Hydrator upgrade tool which caused drafts to not get upgraded. -- `CDAP-4753 `__ - Fixed the Hydrator Stream +- `CDAP-4753 `__ - Fixed the Hydrator Stream source to not assume an output schema. This is valid when a pipeline is created outside Hydrator UI. -- `CDAP-4754 `__ - Fixed ObjectStore to work when +- `CDAP-4754 `__ - Fixed ObjectStore to work when parameterized with custom classes. -- `CDAP-4767 `__ - Fixed an issue where delegation token +- `CDAP-4767 `__ - Fixed an issue where delegation token cancellation of CDAP program was affecting CDAP master services. -- `CDAP-4770 `__ - Fixed the Cask Hydrator UI to +- `CDAP-4770 `__ - Fixed the Cask Hydrator UI to automatically reconnect with the CDAP backend when the backend restarts. -- `CDAP-4771 `__ - Fixed an issue in Cloudera +- `CDAP-4771 `__ - Fixed an issue in Cloudera Manager installations where CDAP container logs would go to the stdout file instead of the master log. -- `CDAP-4784 `__ - Fixed an issue where the +- `CDAP-4784 `__ - Fixed an issue where the IndexedTable was dropping indices upon row updates. -- `CDAP-4785 `__ - Fixed a problem in the upgrade +- `CDAP-4785 `__ - Fixed a problem in the upgrade tool where deleted datasets would cause it to throw a NullPointerException. -- `CDAP-4790 `__ - Fixed an issue where the Hbase +- `CDAP-4790 `__ - Fixed an issue where the Hbase implementation of the Table API returned all rows, when the correct response should have been an empty set of columns. -- `CDAP-4800 `__ - Fixed a problem with the error +- `CDAP-4800 `__ - Fixed a problem with the error message returned when loading an artifact with an invalid range. -- `CDAP-4806 `__ - Fixed the PartitionedFileSet's +- `CDAP-4806 `__ - Fixed the PartitionedFileSet's DynamicPartitioner to work with Avro OutputFormats. -- `CDAP-4829 `__ - Fixed a Validator Transform +- `CDAP-4829 `__ - Fixed a Validator Transform function generator in the Hydrator UI. -- `CDAP-4831 `__ - Allows user-scoped plugins to +- `CDAP-4831 `__ - Allows user-scoped plugins to surface the correct widget JSON in the Hydrator UI. -- `CDAP-4832 `__ - Added the ErrorDataset as an +- `CDAP-4832 `__ - Added the ErrorDataset as an option on widget JSON in Hydrator plugins. -- `CDAP-4836 `__ - Fixed a spacing issue for +- `CDAP-4836 `__ - Fixed a spacing issue for metrics showing in Pipeline diagrams of the Hydrator UI. -- `CDAP-4853 `__ - Fixed issues with the Hydrator +- `CDAP-4853 `__ - Fixed issues with the Hydrator UI widgets for the Hydrator Kafka real-time source, JMS real-time source, and CloneRecord transform. -- `CDAP-4865 `__ - Enhanced the CDAP SDK to be +- `CDAP-4865 `__ - Enhanced the CDAP SDK to be able to publish metadata updates to an external Kafka, identified by the configuration property ``metadata.updates.kafka.broker.list``. Publishing can be enabled by setting ``metadata.updates.publish.enabled`` to true. Updates are published to the Kafka topic identified by the property ``metadata.updates.kafka.topic``. -- `CDAP-4877 `__ - Fixed errors in Cask Hydrator +- `CDAP-4877 `__ - Fixed errors in Cask Hydrator Plugins. Two plugin documents (``core-plugins/docs/Database-batchsink.md`` and ``core-plugins/docs/Database-batchsource.md``) were removed, as the plugins have been moved from *core-plugins* to *database-plugins* (to ``database-plugins/docs/Database-batchsink.md`` and ``database-plugins/docs/Database-batchsource.md``). -- `CDAP-4889 `__ - Fixed an issue with upgrading +- `CDAP-4889 `__ - Fixed an issue with upgrading HBase tables while using the CDAP Upgrade Tool. -- `CDAP-4894 `__ - Fixed an issue with CDAP +- `CDAP-4894 `__ - Fixed an issue with CDAP coprocessors that caused HBase tables to be disabled after upgrading the cluster to a highly-available file system. -- `CDAP-4906 `__ - Fixed the CDAP Upgrade Tool to +- `CDAP-4906 `__ - Fixed the CDAP Upgrade Tool to return a non-zero exit status upon error during upgrade. -- `CDAP-4924 `__ - Fixed a PermGen memory leak +- `CDAP-4924 `__ - Fixed a PermGen memory leak that occurred while deploying multiple applications with database plugins. -- `CDAP-4927 `__ - Fixed the CDAP Explore +- `CDAP-4927 `__ - Fixed the CDAP Explore Service JDBC driver to do nothing instead of throwing an exception when a commit is called. -- `CDAP-4950 `__ - Added an ``'enableAutoCommit'`` +- `CDAP-4950 `__ - Added an ``'enableAutoCommit'`` property to the Cask Hydrator database plugins to enable the use of JDBC drivers that, similar to the Hive JDBC driver, do not allow commits. -- `CDAP-4951 `__ - Changed the upload timeout from the +- `CDAP-4951 `__ - Changed the upload timeout from the CDAP CLI from 15 seconds to unlimited. -- `CDAP-4975 `__ - Pass ResourceManager delegation tokens +- `CDAP-4975 `__ - Pass ResourceManager delegation tokens in the proper format in secure Hadoop HA clusters. Deprecated and Removed Features @@ -4748,67 +4748,67 @@ Known Issues be lost and *WARN* log messages will be logged that tell about the inability to process old data in the old format. -- `CDAP-797 `__ - +- `CDAP-797 `__ - When running secure Hadoop clusters, debug logs from MapReduce programs are not available. -- `CDAP-1007 `__ - +- `CDAP-1007 `__ - If the Hive Metastore is restarted while the CDAP Explore Service is running, the Explore Service remains alive, but becomes unusable. To correct, restart the CDAP Master |---| which will restart all services |---| as described under "Starting CDAP Services" for your particular Hadoop distribution in the `Installation documentation `__. -- `CDAP-1587 `__ - +- `CDAP-1587 `__ - CDAP internally creates tables in the "user" space that begin with the word ``"system"``. User datasets with names starting with ``"system"`` can conflict if they were to match one of those names. To avoid this, do not start any datasets with the word ``"system"``. -- `CDAP-2632 `__ - +- `CDAP-2632 `__ - The application in the `cdap-kafka-ingest-guide `__ does not run on Ubuntu 14.x as of CDAP 3.0.x. -- `CDAP-2721 `__ - +- `CDAP-2721 `__ - Metrics for `FileSets `__ can show zero values even if there is data present, because FileSets do not emit metrics (`CDAP-587 - `). + `). -- `CDAP-2831 `__ - +- `CDAP-2831 `__ - A workflow that is scheduled by time will not be run between the failure of the primary master and the time that the secondary takes over. This scheduled run will not be triggered at all. -- `CDAP-2945 `__ - +- `CDAP-2945 `__ - If the input partition filter for a PartitionedFileSet does not match any partitions, MapReduce jobs can fail. -- `CDAP-3000 `__ - +- `CDAP-3000 `__ - The Workflow token is in an inconsistent state for nodes in a fork while the nodes of the fork are still running. It becomes consistent after the join. -- `CDAP-3221 `__ - +- `CDAP-3221 `__ - When running in Standalone CDAP, mode, if a MapReduce job fails repeatedly, then the SDK hits an out-of-memory exception due to ``perm gen``. The Standalone needs restarting at this point. -- `CDAP-3262 `__ - +- `CDAP-3262 `__ - For Microsoft Windows, the Standalone CDAP scripts can fail when used with a JAVA_HOME that is defined as a path with spaces in it. A workaround is to use a definition of JAVA_HOME that does not include spaces, such as ``C:\PROGRA~1\Java\jdk1.7.0_79\bin`` or ``C:\ProgramData\Oracle\Java\javapath``. -- `CDAP-3492 `__ - +- `CDAP-3492 `__ - In the CDAP CLI, executing ``select *`` from a dataset with many fields generates an error. -- `CDAP-3641 `__ - +- `CDAP-3641 `__ - A RESTful API call to retrieve workflow statistics hangs if units (such as "s" for seconds) are not provided as part of the query. -- `CDAP-3750 `__ - +- `CDAP-3750 `__ - If a table schema contains a field name that is a reserved word in the Hive DDL, ``'enable explore'`` fails. @@ -4817,250 +4817,250 @@ Known Issues New Features ------------ -- `CDAP-961 `__ - +- `CDAP-961 `__ - Added on demand (dynamic) dataset instantiation through program runtime context. -- `CDAP-2303 `__ - +- `CDAP-2303 `__ - Added lookup capability in context that can be used in existing Script, ScriptFilter and Validator transforms. -- `CDAP-3514 `__ - +- `CDAP-3514 `__ - Added an endpoint to get a count of active queries: ``/v3/namespaces//data/explore/queries/count``. -- `CDAP-3857 `__ - +- `CDAP-3857 `__ - Added experimental support for running ETL Batch applications on Spark. Introduced an 'engine' setting in the configuration that defaults to ``'mapreduce'``, but can be set to ``'spark'``. -- `CDAP-3944 `__ - +- `CDAP-3944 `__ - Added support to PartitionConsumer for concurrency, plus a limit and filter on read. -- `CDAP-3945 `__ - +- `CDAP-3945 `__ - Added support for limiting the number of concurrent schedule runs. -- `CDAP-4016 `__ - +- `CDAP-4016 `__ - Added Java-8 support for Script transforms. -- `CDAP-4022 `__ - +- `CDAP-4022 `__ - Added RESTful APIs to start or stop multiple programs. -- `CDAP-4023 `__ - +- `CDAP-4023 `__ - Added CLI commands to stop, start, restart, or get status of programs in an application. -- `CDAP-4043 `__ - +- `CDAP-4043 `__ - Added support for ETL transforms written in Python. -- `CDAP-4128 `__ - +- `CDAP-4128 `__ - Added a new JavaScript transform that can emit records using an emitter. -- `CDAP-4135 `__ - +- `CDAP-4135 `__ - Added the capability for MapReduce and Spark programs to localize additional resources during setup. -- `CDAP-4228 `__ - +- `CDAP-4228 `__ - Added the ability to configure which artifact a Hydrator plugin should use. -- `CDAP-4230 `__ - +- `CDAP-4230 `__ - Added DAGs to ETL pipelines, which will allow users to fork and merge. ETLConfig has been updated to allow representing a DAG. -- `CDAP-4235 `__ - +- `CDAP-4235 `__ - Added AuthorizationPlugin, for pluggable authorization. -- `CDAP-4263 `__ - +- `CDAP-4263 `__ - Added metadata support for stream views. -- `CDAP-4270 `__ - +- `CDAP-4270 `__ - Added CLI support for metadata and lineage. -- `CDAP-4280 `__ - +- `CDAP-4280 `__ - Added the ability to add metadata to artifacts. -- `CDAP-4289 `__ - +- `CDAP-4289 `__ - Added RESTful APIs to set and get properties for an artifact. -- `CDAP-4264 `__ - +- `CDAP-4264 `__ - Added support for automatically annotating CDAP entities with system metadata when they are created or updated. -- `CDAP-4285 `__ - +- `CDAP-4285 `__ - Added an authorization plugin that uses a system dataset to manage ACLs. -- `CDAP-4403 `__ - +- `CDAP-4403 `__ - Moved Hydrator plugins from the CDAP repository as cdap-etl-lib into its own repository. -- `CDAP-4591 `__ - +- `CDAP-4591 `__ - Improved Metadata Indexing and Search to support searches on words in value and tags. -- `CDAP-4592 `__ - +- `CDAP-4592 `__ - Schema fields are stored as Metadata and are searchable. -- `CDAP-4658 `__ - +- `CDAP-4658 `__ - Added capability in CDAP UI to display system tags. Improvements ------------ -- `CDAP-3079 `__ - +- `CDAP-3079 `__ - Table datasets, and any other dataset that implements ``RecordWritable``, can now be written to using Hive. -- `CDAP-3887 `__ - +- `CDAP-3887 `__ - The CDAP Router now has a configurable timeout for idle connections, with a default timeout of 15 seconds. -- `CDAP-4045 `__ - +- `CDAP-4045 `__ - A new property master.collect.containers.log has been added to cdap-site.xml, which determines if container logs are streamed back to the cdap-master process log. (This has always been the default behavior). For MapR installations, this must be turned off (set to false). -- `CDAP-4133 `__ - +- `CDAP-4133 `__ - Added ability to retrieve the live-info for the AppFabric system service. -- `CDAP-4209 `__ - +- `CDAP-4209 `__ - Added a method to ``ObjectMappedTable`` and ``ObjectStore`` to retrieve a specific number of splits between a start and end keys. -- `CDAP-4233 `__ - +- `CDAP-4233 `__ - Messages logged by Hydrator are now prefixed with the name of the stage that logged them. -- `CDAP-4301 `__ - +- `CDAP-4301 `__ - Added support for CDH5.5 -- `CDAP-4392 `__ - +- `CDAP-4392 `__ - Upgraded netty-http dependency in CDAP to 0.14.0. -- `CDAP-4444 `__ - +- `CDAP-4444 `__ - Make ``xmllint`` dependency optional and allow setting variables to skip configuration file parsing. -- `CDAP-4453 `__ - +- `CDAP-4453 `__ - Added a schema validation |---| for sources, transforms, and sinks |---| that will validate the pipeline stages schema during deployment, and report any issues. -- `CDAP-4518 `__ - +- `CDAP-4518 `__ - CDAP Master service will now log important configuration settings on startup. -- `CDAP-4523 `__ - +- `CDAP-4523 `__ - Added the config setting ``master.startup.checks.enabled`` to control whether CDAP Master startup checks are run or not. -- `CDAP-4536 `__ - +- `CDAP-4536 `__ - Improved the installation experience by adding to the CDAP Master service checks of pre-requisites such as file system permissions, availability of components such as YARN and HBase, resource availability during startup, and to error out if any of the pre-requisites fail. -- `CDAP-4548 `__ - +- `CDAP-4548 `__ - Added a config setting 'master.collect.app.containers.log' that can be set to 'false' to disable streaming of application logs back to the CDAP Master log. -- `CDAP-4598 `__ - +- `CDAP-4598 `__ - Added an error message when a required field is not provided when configuring Hydrator pipeline. Bug Fixes --------- -- `CDAP-1174 `__ - +- `CDAP-1174 `__ - Prefix start script functions with ``'cdap'`` to prevent namespace collisions. -- `CDAP-2470 `__ - +- `CDAP-2470 `__ - Added a check to cause a DB (source or sink) pipeline to fail during deployment if the table (source or sink) was not found, or if an incorrect connection string was provided. -- `CDAP-3345 `__ - +- `CDAP-3345 `__ - Fixed a bug where the TTL for datasets was incorrect; it was reduced by (a factor of 1000) after an upgrade. After running the upgrade tool, please make sure the TTL values of tables are as expected. -- `CDAP-3542 `__ - +- `CDAP-3542 `__ - Fixed an issue where the failure of a program running in a workflow fork node was causing other programs in the same fork node to remain in the RUNNING state, even after the Workflow was completed. -- `CDAP-3694 `__ - +- `CDAP-3694 `__ - Fixed test failures in the PurchaseHistory, StreamConversion, and WikipediaPipeline example apps included in the CDAP SDK. -- `CDAP-3742 `__ - +- `CDAP-3742 `__ - Fixed a bug where certain MapReduce metrics were not being properly emitted when using multiple outputs. -- `CDAP-3761 `__ - +- `CDAP-3761 `__ - Fixed a problem with DBSink column names not being used to filter input record fields before writing to a DBSink. -- `CDAP-3807 `__ - +- `CDAP-3807 `__ - Added a fix for case sensitivity handling in DBSink. -- `CDAP-3815 `__ - +- `CDAP-3815 `__ - Fixed an issue where the regex filter for S3 Batch Source wasn't getting applied correctly. -- `CDAP-3861 `__ - +- `CDAP-3861 `__ - Fixed an issue about stopping all dependent services when a service is stopped. -- `CDAP-3900 `__ - +- `CDAP-3900 `__ - Fixed a bug when querying for logs of deleted program runs. -- `CDAP-3902 `__ - +- `CDAP-3902 `__ - Fixed a problem with dataset performance degradation because of making multiple remote calls for each "get dataset" request. -- `CDAP-3924 `__ - +- `CDAP-3924 `__ - Fixed QueryClient to work against HTTPS. -- `CDAP-4000 `__ - +- `CDAP-4000 `__ - Fixed an issue where a stream that has a view could not be deleted cleanly. -- `CDAP-4067 `__ - +- `CDAP-4067 `__ - Fixed an issue where socket connections to the TransactionManager were not being closed. -- `CDAP-4092 `__ - +- `CDAP-4092 `__ - Fixes an issue that causes worker threads to go into an infinite recursion while exceptions are being thrown in channel handlers. -- `CDAP-4112 `__ - +- `CDAP-4112 `__ - Fixed a bug that prevented applications from using HBase directly. -- `CDAP-4119 `__ - +- `CDAP-4119 `__ - Fixed a problem where when CDAP Master switched from active to standby, the programs that were running were marked as failed. -- `CDAP-4240 `__ - +- `CDAP-4240 `__ - Fixed a problem in the CLI command used to load an artifact, where the wrong artifact name and version was used if the artifact name ends with a number. -- `CDAP-4294 `__ - +- `CDAP-4294 `__ - Fixed a problem where plugins from another namespace were visible when creating an application using a system artifact. -- `CDAP-4316 `__ - +- `CDAP-4316 `__ - Fixed a problem with the CLI attempting to connect to CDAP when the hostname and port were incorrect. -- `CDAP-4366 `__ - +- `CDAP-4366 `__ - Improved error message when stream views were not found. -- `CDAP-4393 `__ - +- `CDAP-4393 `__ - Fixed an issue where tags search were failing for certain tags. -- `CDAP-4141 `__ - +- `CDAP-4141 `__ - Fixed node.js version checking for the ``cdap sdk`` script in the CDAP SDK. -- `CDAP-4373 `__ - +- `CDAP-4373 `__ - Fixed a problem that prevented MapReduce jobs from being run when the Resource Manager switches from active to standby in a Kerberos-enabled HA cluster. -- `CDAP-4384 `__ - +- `CDAP-4384 `__ - Fixed an issue that prevents streams from being read in HA HDFS mode. -- `CDAP-4526 `__ - +- `CDAP-4526 `__ - Fixed init scripts to print service status when stopped. -- `CDAP-4534 `__ - +- `CDAP-4534 `__ - Added configuration 'router.bypass.auth.regex' to exempt certain URLs from authentication. -- `CDAP-4539 `__ - +- `CDAP-4539 `__ - Fixed a problem in the init scripts that forced ``cdap-kafka-server``, ``cdap-router``, and ``cdap-auth-server`` to have the Hive client installed. -- `CDAP-4678 `__ - +- `CDAP-4678 `__ - Fixed an issue where the logs and history list on a Hydrator pipeline view was not updating on new runs. @@ -5070,20 +5070,20 @@ Deprecated and Removed Features - See the `CDAP 3.3.0 Javadocs `__ for a list of deprecated and removed APIs. -- `CDAP-2481 `__ - +- `CDAP-2481 `__ - Removed a deprecated endpoint to retrieve the status of a currently running node in a workflow. -- `CDAP-2943 `__ - +- `CDAP-2943 `__ - Removed the deprecated builder-style Flow API. -- `CDAP-4128 `__ - +- `CDAP-4128 `__ - Deprecated the Script transform. -- `CDAP-4217 `__ - +- `CDAP-4217 `__ - Deprecated createDataSchedule and createTimeSchedule methods in Schedules class and removed deprecated Schedule constructor. -- `CDAP-4251 `__ - +- `CDAP-4251 `__ - Removed deprecated fluent style API for Flow configuration. The only supported API is now the configurer style. Known Issues @@ -5092,67 +5092,67 @@ Known Issues be lost and *WARN* log messages will be logged that tell about the inability to process old data in the old format. -- `CDAP-797 `__ - +- `CDAP-797 `__ - When running secure Hadoop clusters, debug logs from MapReduce programs are not available. -- `CDAP-1007 `__ - +- `CDAP-1007 `__ - If the Hive Metastore is restarted while the CDAP Explore Service is running, the Explore Service remains alive, but becomes unusable. To correct, restart the CDAP Master |---| which will restart all services |---| as described under "Starting CDAP Services" for your particular Hadoop distribution in the `Installation documentation `__. -- `CDAP-1587 `__ - +- `CDAP-1587 `__ - CDAP internally creates tables in the "user" space that begin with the word ``"system"``. User datasets with names starting with ``"system"`` can conflict if they were to match one of those names. To avoid this, do not start any datasets with the word ``"system"``. -- `CDAP-2632 `__ - +- `CDAP-2632 `__ - The application in the `cdap-kafka-ingest-guide `__ does not run on Ubuntu 14.x as of CDAP 3.0.x. -- `CDAP-2721 `__ - +- `CDAP-2721 `__ - Metrics for `FileSets `__ can show zero values even if there is data present, because FileSets do not emit metrics (`CDAP-587 - `). + `). -- `CDAP-2831 `__ - +- `CDAP-2831 `__ - A workflow that is scheduled by time will not be run between the failure of the primary master and the time that the secondary takes over. This scheduled run will not be triggered at all. -- `CDAP-2945 `__ - +- `CDAP-2945 `__ - If the input partition filter for a PartitionedFileSet does not match any partitions, MapReduce jobs can fail. -- `CDAP-3000 `__ - +- `CDAP-3000 `__ - The Workflow token is in an inconsistent state for nodes in a fork while the nodes of the fork are still running. It becomes consistent after the join. -- `CDAP-3221 `__ - +- `CDAP-3221 `__ - When running in Standalone CDAP, if a MapReduce job fails repeatedly, then the SDK hits an out-of-memory exception due to ``perm gen``. The Standalone needs restarting at this point. -- `CDAP-3262 `__ - +- `CDAP-3262 `__ - For Microsoft Windows, the Standalone CDAP scripts can fail when used with a JAVA_HOME that is defined as a path with spaces in it. A workaround is to use a definition of JAVA_HOME that does not include spaces, such as ``C:\PROGRA~1\Java\jdk1.7.0_79\bin`` or ``C:\ProgramData\Oracle\Java\javapath``. -- `CDAP-3492 `__ - +- `CDAP-3492 `__ - In the CDAP CLI, executing ``select *`` from a dataset with many fields generates an error. -- `CDAP-3641 `__ - +- `CDAP-3641 `__ - A RESTful API call to retrieve workflow statistics hangs if units (such as "s" for seconds) are not provided as part of the query. -- `CDAP-3750 `__ - +- `CDAP-3750 `__ - If a table schema contains a field name that is a reserved word in the Hive DDL, ``'enable explore'`` fails. @@ -5162,49 +5162,49 @@ Known Issues New Features ------------ -- `CDAP-3951 `__ - +- `CDAP-3951 `__ - Added the ability for S3 batch sources and sinks to set additional file system properties. Improvements ------------ -- `CDAP-3870 `__ - +- `CDAP-3870 `__ - Added logging and metrics support for *Script*, *ScriptFilter*, and *Validator* transforms. -- `CDAP-3939 `__ - +- `CDAP-3939 `__ - Improved artifact and application deployment failure handling. Bug Fixes --------- -- `CDAP-3342 `__ - +- `CDAP-3342 `__ - Fixed a problem with the CDAP SDK unable to start on certain Windows machines by updating the Hadoop native library in CDAP with a version that does not have a dependency on a debug version of the Microsoft ``msvcr100.dll``. -- `CDAP-3815 `__ - +- `CDAP-3815 `__ - Fixed an issue where the regex filter for S3 batch sources wasn't being applied correctly. -- `CDAP-3829 `__ - +- `CDAP-3829 `__ - Fixed snapshot sinks so that the data is explorable as a ``PartitionedFileSet``. -- `CDAP-3833 `__ - +- `CDAP-3833 `__ - Fixed snapshot sinks so that they can be read safely. -- `CDAP-3859 `__ - +- `CDAP-3859 `__ - Fixed a compilation error in the Maven application archetype. -- `CDAP-3860 `__ - +- `CDAP-3860 `__ - Fixed a bug where plugins, packaged in the same artifact as an application class, could not be used by that application class. -- `CDAP-3891 `__ - +- `CDAP-3891 `__ - Updated the documentation to remove references to application templates and adaptors that were removed as of CDAP 3.2.0. -- `CDAP-3949 `__ - +- `CDAP-3949 `__ - Fixed a problem with running certain examples on Linux systems by increasing the maximum Java heap size of the Standalone SDK on Linux systems to 2048m. -- `CDAP-3961 `__ - +- `CDAP-3961 `__ - Fixed a missing dependency on ``cdap-hbase-compat-1.1`` package in the CDAP Master package. Known Issues @@ -5213,67 +5213,67 @@ Known Issues be lost and *WARN* log messages will be logged that tell about the inability to process old data in the old format. -- `CDAP-797 `__ - +- `CDAP-797 `__ - When running secure Hadoop clusters, debug logs from MapReduce programs are not available. -- `CDAP-1007 `__ - +- `CDAP-1007 `__ - If the Hive Metastore is restarted while the CDAP Explore Service is running, the Explore Service remains alive, but becomes unusable. To correct, restart the CDAP Master |---| which will restart all services |---| as described under "Starting CDAP Services" for your particular Hadoop distribution in the `Installation documentation `__. -- `CDAP-1587 `__ - +- `CDAP-1587 `__ - CDAP internally creates tables in the "user" space that begin with the word ``"system"``. User datasets with names starting with ``"system"`` can conflict if they were to match one of those names. To avoid this, do not start any datasets with the word ``"system"``. -- `CDAP-2632 `__ - +- `CDAP-2632 `__ - The application in the `cdap-kafka-ingest-guide `__ does not run on Ubuntu 14.x as of CDAP 3.0.x. -- `CDAP-2721 `__ - +- `CDAP-2721 `__ - Metrics for `FileSets `__ can show zero values even if there is data present, because FileSets do not emit metrics (`CDAP-587 - `). + `). -- `CDAP-2831 `__ - +- `CDAP-2831 `__ - A workflow that is scheduled by time will not be run between the failure of the primary master and the time that the secondary takes over. This scheduled run will not be triggered at all. -- `CDAP-2945 `__ - +- `CDAP-2945 `__ - If the input partition filter for a PartitionedFileSet does not match any partitions, MapReduce jobs can fail. -- `CDAP-3000 `__ - +- `CDAP-3000 `__ - The Workflow token is in an inconsistent state for nodes in a fork while the nodes of the fork are still running. It becomes consistent after the join. -- `CDAP-3221 `__ - +- `CDAP-3221 `__ - When running in Standalone CDAP, if a MapReduce job fails repeatedly, then the SDK hits an out-of-memory exception due to ``perm gen``. The Standalone needs restarting at this point. -- `CDAP-3262 `__ - +- `CDAP-3262 `__ - For Microsoft Windows, the Standalone CDAP scripts can fail when used with a JAVA_HOME that is defined as a path with spaces in it. A workaround is to use a definition of JAVA_HOME that does not include spaces, such as ``C:\PROGRA~1\Java\jdk1.7.0_79\bin`` or ``C:\ProgramData\Oracle\Java\javapath``. -- `CDAP-3492 `__ - +- `CDAP-3492 `__ - In the CDAP CLI, executing ``select *`` from a dataset with many fields generates an error. -- `CDAP-3641 `__ - +- `CDAP-3641 `__ - A RESTful API call to retrieve workflow statistics hangs if units (such as "s" for seconds) are not provided as part of the query. -- `CDAP-3750 `__ - +- `CDAP-3750 `__ - If a table schema contains a field name that is a reserved word in the Hive DDL, ``'enable explore'`` fails. @@ -5283,387 +5283,387 @@ Known Issues New Features ------------ -- `CDAP-2556 `__ - +- `CDAP-2556 `__ - Added support for HBase1.1. -- `CDAP-2666 `__ - +- `CDAP-2666 `__ - Added a new API for creating an application from an artifact. -- `CDAP-2756 `__ - +- `CDAP-2756 `__ - Added the ability to write to multiple outputs from a MapReduce job. -- `CDAP-2757 `__ - +- `CDAP-2757 `__ - Added the ability to dynamically write to multiple partitions of a PartitionedFileSet dataset as the output of a MapReduce job. -- `CDAP-3253 `__ - +- `CDAP-3253 `__ - Added a Stream and Dataset Widget to the CDAP UI. -- `CDAP-3390 `__ - +- `CDAP-3390 `__ - Added stream views, enabling reading from a single stream using various formats and schemas. -- `CDAP-3476 `__ - +- `CDAP-3476 `__ - Added a Validator Transform that can be used to validate records based on a set of available validators and configured to write invalid records to an error dataset. -- `CDAP-3516 `__ - +- `CDAP-3516 `__ - Added a service to manage the metadata of CDAP entities. -- `CDAP-3518 `__ - +- `CDAP-3518 `__ - Added the publishing of metadata change notifications to Apache Kafka. -- `CDAP-3519 `__ - +- `CDAP-3519 `__ - Added the ability to compute lineage of a CDAP dataset or stream in a given time window. -- `CDAP-3520 `__ - +- `CDAP-3520 `__ - Added RESTful APIs for adding/retrieving/deleting of metadata for apps/programs/datasets/streams. -- `CDAP-3521 `__ - +- `CDAP-3521 `__ - Added the ability to record a dataset or stream access by a CDAP program. -- `CDAP-3522 `__ - +- `CDAP-3522 `__ - Added the capability to search CDAP entities based on their metadata. -- `CDAP-3523 `__ - +- `CDAP-3523 `__ - Added RESTful APIs for searching CDAP entities based on business metadata. -- `CDAP-3527 `__ - +- `CDAP-3527 `__ - Added a data store to manage business metadata of CDAP entities. -- `CDAP-3549 `__ - +- `CDAP-3549 `__ - Added SSH port forwarding to the CDAP virtual machine. -- `CDAP-3556 `__ - +- `CDAP-3556 `__ - Added a data store for recording data accesses by CDAP programs and computing lineage. -- `CDAP-3590 `__ - +- `CDAP-3590 `__ - Added the ability to write to multiple sinks in ETL real-time and batch applications. -- `CDAP-3591 `__ - +- `CDAP-3591 `__ - Added the ability for real-time ETL pipelines to write to multiple sinks. -- `CDAP-3592 `__ - +- `CDAP-3592 `__ - Added the ability for batch ETL pipelines to write to multiple sinks. -- `CDAP-3626 `__ - +- `CDAP-3626 `__ - For the CSV and TSV stream formats, a "mapping" setting can now be specified, mapping stream event columns to schema columns. -- `CDAP-3693 `__ - +- `CDAP-3693 `__ - Added support for CDAP to work with HDP 2.3. Improvements ------------ -- `CDAP-1914 `__ - +- `CDAP-1914 `__ - Added documentation of the RESTful endpoint to retrieve the properties of a stream. -- `CDAP-2514 `__ - +- `CDAP-2514 `__ - Added an interface to load a file into a stream from the CDAP UI. -- `CDAP-2809 `__ - +- `CDAP-2809 `__ - The CDAP UI "Errors" pop-up in the main screen now displays the time and date for each error. -- `CDAP-2872 `__ - +- `CDAP-2872 `__ - Updated the Cloudera Manager CSD to use support for logback. -- `CDAP-2950 `__ - +- `CDAP-2950 `__ - Cleaned up the messages shown in the errors dropdown in the CDAP UI. -- `CDAP-3147 `__ - +- `CDAP-3147 `__ - Added a CDAP CLI command to stop a workflow. -- `CDAP-3179 `__ - +- `CDAP-3179 `__ - Added support for upgrading the Hadoop distribution or the HBase version that CDAP is running on. -- `CDAP-3257 `__ - +- `CDAP-3257 `__ - Revised the documentation of the file ``cdap-default.xml``, removed properties no longer in use, and corrected discrepancies between the documentation and the shipped XML file. -- `CDAP-3270 `__ - +- `CDAP-3270 `__ - Improved the help provided in the CDAP CLI for the setting of stream formats. -- `CDAP-3275 `__ - +- `CDAP-3275 `__ - Upgraded netty-http version to 0.12.0. -- `CDAP-3282 `__ - +- `CDAP-3282 `__ - Added a HTTP RESTful API to update the application configuration and artifact version. -- `CDAP-3332 `__ - +- `CDAP-3332 `__ - Added a "clear" button in the CDAP UI for cases where a user decides to not used a pre-populated schema. -- `CDAP-3351 `__ - +- `CDAP-3351 `__ - Defined a directory structure to be used for predefined applications. -- `CDAP-3357 `__ - +- `CDAP-3357 `__ - Added documentation in the source code on adding new commands and completers to the CDAP CLI. -- `CDAP-3393 `__ - +- `CDAP-3393 `__ - In the CDAP UI, added visualization for Workflow tokens in Workflows. -- `CDAP-3419 `__ - +- `CDAP-3419 `__ - HBaseQueueDebugger now shows the minimum queue event transaction write pointer both for each queue and for all queues. -- `CDAP-3443 `__ - +- `CDAP-3443 `__ - Added an example cdap-env.sh to the shipped packages. -- `CDAP-3464 `__ - +- `CDAP-3464 `__ - Added an example in the documentation explaining how to prune invalid transactions from the transaction manager. -- `CDAP-3490 `__ - +- `CDAP-3490 `__ - Modified the CDAP upgrade tool to delete all adapters and the ETLBatch and ETLRealtime ApplicationTemplates. -- `CDAP-3495 `__ - +- `CDAP-3495 `__ - Added the ability to persist the runtime arguments with which a program was run. -- `CDAP-3550 `__ - +- `CDAP-3550 `__ - Added support for writing to Amazon S3 in Avro and Parquet formats from batch ETL applications. -- `CDAP-3564 `__ - +- `CDAP-3564 `__ - Updated CDAP to use Tephra 0.6.2. -- `CDAP-3610 `__ - +- `CDAP-3610 `__ - Updated the transaction debugger client to print checkpoint information. Bug Fixes --------- -- `CDAP-1697 `__ - +- `CDAP-1697 `__ - Fixed an issue where failed dataset operations via Explore queries did not invalidate the associated transaction. -- `CDAP-1864 `__ - +- `CDAP-1864 `__ - Fixed a problem where users got an incorrect message while creating a dataset in a non-existent namespace. -- `CDAP-1892 `__ - +- `CDAP-1892 `__ - Fixed a problem with services returning the same message for all failures. -- `CDAP-1984 `__ - +- `CDAP-1984 `__ - Fixed a problem where a dataset could be created in a non-existent namespace in standalone mode. -- `CDAP-2428 `__ - +- `CDAP-2428 `__ - Fixed a problem with the CDAP CLI creating file logs. -- `CDAP-2521 `__ - +- `CDAP-2521 `__ - Fixed a problem with the CDAP CLI not auto-completing when setting a stream format. -- `CDAP-2785 `__ - +- `CDAP-2785 `__ - Fixed a problem with the CDAP UI of buttons staying 'in focus' after clicking. -- `CDAP-2809 `__ - +- `CDAP-2809 `__ - The CDAP UI "Errors" pop-up in the main screen now displays the time and date for each error. -- `CDAP-2892 `__ - +- `CDAP-2892 `__ - Fixed a problem with schedules not being deployed in suspended mode. -- `CDAP-3014 `__ - +- `CDAP-3014 `__ - Fixed a problem where failure of a spark node would cause a workflow to restart indefinitely. -- `CDAP-3073 `__ - +- `CDAP-3073 `__ - Fixed an issue with the Standalone CDAP process periodically crashing with Out-of-Memory errors when writing to an Oracle table. -- `CDAP-3101 `__ - +- `CDAP-3101 `__ - Fixed a problem with workflow runs not getting scheduled due to Quartz exceptions. -- `CDAP-3121 `__ - +- `CDAP-3121 `__ - Fixed a problem with discrepancies between the documentation and the defaults actually used by CDAP. -- `CDAP-3200 `__ - +- `CDAP-3200 `__ - Fixed a problem in the CDAP UI with the clone button in an incorrect position when using Firefox. -- `CDAP-3201 `__ - +- `CDAP-3201 `__ - Fixed a problem in the CDAP UI with an incorrect tabbing order when using Firefox. -- `CDAP-3219 `__ - +- `CDAP-3219 `__ - Fixed a problem when specifying the HBase version using the HBASE_VERSION environment variable. -- `CDAP-3233 `__ - +- `CDAP-3233 `__ - Fixed a problem in the CDAP UI error pop-ups not having a default focus on a button. -- `CDAP-3243 `__ - +- `CDAP-3243 `__ - Fixed a problem in the CDAP UI with the default schema shown for streams. -- `CDAP-3260 `__ - +- `CDAP-3260 `__ - Fixed a problem in the CDAP UI with scrolling on the namespaces dropdown on certain pages. -- `CDAP-3261 `__ - +- `CDAP-3261 `__ - Fixed a problem on Distributed CDAP with the serializing of the metadata artifact causing a stack overflow. -- `CDAP-3305 `__ - +- `CDAP-3305 `__ - Fixed a problem in the CDAP UI not warning users if they exit or close their browser without saving. -- `CDAP-3313 `__ - +- `CDAP-3313 `__ - Fixed a problem in the CDAP UI with refreshing always returning to the overview page. -- `CDAP-3326 `__ - +- `CDAP-3326 `__ - Fixed a problem with the table batch source requiring a row key to be set. -- `CDAP-3343 `__ - +- `CDAP-3343 `__ - Fixed a problem with the application deployment for apps that contain Spark. -- `CDAP-3349 `__ - +- `CDAP-3349 `__ - Fixed a problem with the display of ETL application metrics in the CDAP UI. -- `CDAP-3355 `__ - +- `CDAP-3355 `__ - Fixed a problem in the CDAP examples with the use of a runtime argument, ``min.pages.threshold``. -- `CDAP-3362 `__ - +- `CDAP-3362 `__ - Fixed a problem with the ``logback-container.xml`` not being copied into master services. -- `CDAP-3374 `__ - +- `CDAP-3374 `__ - Fixed a problem with warning messages in the logs indicating that programs were running that actually were not running. -- `CDAP-3376 `__ - +- `CDAP-3376 `__ - Fixed a problem with being unable to deploy the SparkPageRank example application on a cluster. -- `CDAP-3386 `__ - +- `CDAP-3386 `__ - Fixed a problem with the Spark classes not being found when running a Spark program through a Workflow in Distributed CDAP on HDP 2.2. -- `CDAP-3394 `__ - +- `CDAP-3394 `__ - Fixed a problem with the deployment of applications through the CDAP UI. -- `CDAP-3399 `__ - +- `CDAP-3399 `__ - Fixed a problem with the SparkPageRankApp example spawning multiple containers in distributed mode due to its number of services. -- `CDAP-3400 `__ - +- `CDAP-3400 `__ - Fixed an issue with warning messages about the notification system every time the CDAP Standalone is restarted. -- `CDAP-3408 `__ - +- `CDAP-3408 `__ - Fixed a problem with running the CDAP Explore Service on CDH 5.[2,3]. -- `CDAP-3432 `__ - +- `CDAP-3432 `__ - Fixed a bug where connecting with a certain namespace from the CLI would not immediately display that namespace in the CLI prompt. -- `CDAP-3435 `__ - +- `CDAP-3435 `__ - Fixed an issue where the program status was shown as running even after it is stopped. -- `CDAP-3442 `__ - +- `CDAP-3442 `__ - Fixed a problem that caused application creation to fail if a config setting was given to an application that does not use a config. -- `CDAP-3449 `__ - +- `CDAP-3449 `__ - Fixed a problem with the readless increment co-processor not handling multiple readless increment columns in the same row. -- `CDAP-3452 `__ - +- `CDAP-3452 `__ - Fixed a problem that prevented explore service working on clusters with secure hive 0.14. -- `CDAP-3458 `__ - +- `CDAP-3458 `__ - Fixed a problem where streams events that had already been processed were re-processed in flows. -- `CDAP-3470 `__ - +- `CDAP-3470 `__ - Fixed an issue with error messages being logged during a master process restart. -- `CDAP-3472 `__ - +- `CDAP-3472 `__ - Fixed the error message returned when trying to stop a program started by a workflow. -- `CDAP-3473 `__ - +- `CDAP-3473 `__ - Fixed a problem with a workflow failure not updating a run record for the inner program. -- `CDAP-3530 `__ - +- `CDAP-3530 `__ - Fixed a problem with the CDAP UI performance when rendering flow diagrams with a large number of nodes. -- `CDAP-3563 `__ - +- `CDAP-3563 `__ - Removed faulty and unused metrics around CDAP file resource usage. -- `CDAP-3574 `__ - +- `CDAP-3574 `__ - Fix an issue with Explore not working on HDP Hive 0.12. -- `CDAP-3603 `__ - +- `CDAP-3603 `__ - Fixed an issue with configuration properties for ETL Transforms being validated at runtime instead of when an application is created. -- `CDAP-3618 `__ - +- `CDAP-3618 `__ - Fix a problem where suspended schedules were lost when CDAP master was restarted. -- `CDAP-3660 `__ - +- `CDAP-3660 `__ - Fixed and issue where the Hadoop filesystem object was getting instantiated before the Kerberos keytab login was completed, leading to CDAP processes failing after the initial ticket expired. -- `CDAP-3700 `__ - +- `CDAP-3700 `__ - Fixed an issue with the log saver having numerous open connections to HBase, causing it to go Out-of-Memory. -- `CDAP-3711 `__ - +- `CDAP-3711 `__ - Fixed an issue that prevented the downloading of Explore results on a secure cluster. -- `CDAP-3713 `__ - +- `CDAP-3713 `__ - Fixed an issue where certain RESTful APIs were not returning appropriate error messages for internal server errors. -- `CDAP-3716 `__ - +- `CDAP-3716 `__ - Fixed a possible deadlock when CDAP master is restarted with an existing app running on a cluster. API Changes ----------- -- `CDAP-2763 `__ - +- `CDAP-2763 `__ - Added RESTful APIs for managing artifacts. -- `CDAP-2956 `__ - +- `CDAP-2956 `__ - Deprecated the existing API for configuring a workflow action, replacing it with a simpler API. -- `CDAP-3063 `__ - +- `CDAP-3063 `__ - Added CLI commands for managing artifacts. -- `CDAP-3064 `__ - +- `CDAP-3064 `__ - Added an ArtifactClient to interact with Artifact HTTP RESTful APIs. -- `CDAP-3283 `__ - +- `CDAP-3283 `__ - Added artifact information to Application RESTful APIs and the means to filter applications by artifact name and version. -- `CDAP-3324 `__ - +- `CDAP-3324 `__ - Added a RESTful API for creating an application from an artifact. -- `CDAP-3367 `__ - +- `CDAP-3367 `__ - Added the ability to delete an artifact. -- `CDAP-3488 `__ - +- `CDAP-3488 `__ - Changed the ETLBatchTemplate from an ApplicationTemplate to an Application. -- `CDAP-3535 `__ - +- `CDAP-3535 `__ - Added an API for programs to retrieve their application specification at runtime. -- `CDAP-3554 `__ - +- `CDAP-3554 `__ - Changed the plugin types from 'source' to either 'batchsource' or 'realtimesource', and from 'sink' to either 'batchsink' or 'realtimesink' to reflect that the plugins implement different interfaces. -- `CDAP-1554 `__ - +- `CDAP-1554 `__ - Moved constants for default and system namespaces from Common to Id. -- `CDAP-3388 `__ - +- `CDAP-3388 `__ - Added interfaces to ``cdap-spi`` that abstract StreamEventRecordFormat (and dependent interfaces) so users can extend the ``cdap-spi`` interfaces. -- `CDAP-3583 `__ - +- `CDAP-3583 `__ - Added a RESTful API for retrieving the metadata associated with a particular run of a CDAP program. -- `CDAP-3632 `__ - +- `CDAP-3632 `__ - Added a RESTful API for computing lineage of a CDAP dataset or stream. Deprecated and Removed Features @@ -5673,16 +5673,16 @@ Deprecated and Removed Features `__ for a list of deprecated and removed APIs. -- `CDAP-2667 `__ - +- `CDAP-2667 `__ - Removed application templates and adapters RESTful APIs, as these templates and adapters have been replaced with applications that can be controlled with the `Lifecycle HTTP RESTful API `__. -- `CDAP-2951 `__ - +- `CDAP-2951 `__ - Removed deprecated methods in cdap-client. -- `CDAP-3596 `__ - +- `CDAP-3596 `__ - Replaced the ETL ApplicationTemplates with the new ETL Applications. Known Issues @@ -5691,79 +5691,79 @@ Known Issues be lost and *WARN* log messages will be logged that tell about the inability to process old data in the old format. -- `CDAP-797 `__ - +- `CDAP-797 `__ - When running secure Hadoop clusters, debug logs from MapReduce programs are not available. -- `CDAP-1007 `__ - +- `CDAP-1007 `__ - If the Hive Metastore is restarted while the CDAP Explore Service is running, the Explore Service remains alive, but becomes unusable. To correct, restart the CDAP Master |---| which will restart all services |---| as described under "Starting CDAP Services" for your particular Hadoop distribution in the `Installation documentation `__. -- `CDAP-1587 `__ - +- `CDAP-1587 `__ - CDAP internally creates tables in the "user" space that begin with the word ``"system"``. User datasets with names starting with ``"system"`` can conflict if they were to match one of those names. To avoid this, do not start any datasets with the word ``"system"``. -- `CDAP-2632 `__ - +- `CDAP-2632 `__ - The application in the `cdap-kafka-ingest-guide `__ does not run on Ubuntu 14.x as of CDAP 3.0.x. -- `CDAP-2721 `__ - +- `CDAP-2721 `__ - Metrics for `FileSets `__ can show zero values even if there is data present, because FileSets do not emit metrics (`CDAP-587 - `). + `). -- `CDAP-2831 `__ - +- `CDAP-2831 `__ - A workflow that is scheduled by time will not be run between the failure of the primary master and the time that the secondary takes over. This scheduled run will not be triggered at all. -- `CDAP-2945 `__ - +- `CDAP-2945 `__ - If the input partition filter for a PartitionedFileSet does not match any partitions, MapReduce jobs can fail. -- `CDAP-3000 `__ - +- `CDAP-3000 `__ - The Workflow token is in an inconsistent state for nodes in a fork while the nodes of the fork are still running. It becomes consistent after the join. -- `CDAP-3221 `__ - +- `CDAP-3221 `__ - When running in Standalone CDAP, if a MapReduce job fails repeatedly, then the SDK hits an out-of-memory exception due to ``perm gen``. The Standalone needs restarting at this point. -- `CDAP-3262 `__ - +- `CDAP-3262 `__ - For Microsoft Windows, the Standalone CDAP scripts can fail when used with a JAVA_HOME that is defined as a path with spaces in it. A workaround is to use a definition of JAVA_HOME that does not include spaces, such as ``C:\PROGRA~1\Java\jdk1.7.0_79\bin`` or ``C:\ProgramData\Oracle\Java\javapath``. -- `CDAP-3492 `__ - +- `CDAP-3492 `__ - In the CDAP CLI, executing ``select *`` from a dataset with many fields generates an error. -- `CDAP-3641 `__ - +- `CDAP-3641 `__ - A RESTful API call to retrieve workflow statistics hangs if units (such as "s" for seconds) are not provided as part of the query. -- `CDAP-3697 `__ - +- `CDAP-3697 `__ - CDAP Explore is broken on secure CDH 5.1. -- `CDAP-3698 `__ - +- `CDAP-3698 `__ - CDAP Explore is unable to get a delegation token while fetching next results on HDP2.0. -- `CDAP-3749 `__ - +- `CDAP-3749 `__ - The DBSource plugin does not allow a username with an empty password. -- `CDAP-3750 `__ - +- `CDAP-3750 `__ - If a table schema contains a field name that is a reserved word in the Hive DDL, ``'enable explore'`` fails. -- `CDAP-3819 `__ - +- `CDAP-3819 `__ - The Cassandra source does not handles spaces properly in column fields which require a comma-separated list. @@ -5775,127 +5775,127 @@ New Features **MapR 4.1 Support, HDP 2.2 Support, CDH 5.4 Support** -- `CDAP-1614 `__ - +- `CDAP-1614 `__ - Added HBase 1.0 support. -- `CDAP-2318 `__ - +- `CDAP-2318 `__ - Made CDAP work on the HDP 2.2 distribution. -- `CDAP-2786 `__ - +- `CDAP-2786 `__ - Added support to CDAP 3.1.0 for the MapR 4.1 distro. -- `CDAP-2798 `__ - +- `CDAP-2798 `__ - Added Hive 0.14 support. -- `CDAP-2801 `__ - +- `CDAP-2801 `__ - Added CDH 5.4 Hive 1.1 support. -- `CDAP-2836 `__ - +- `CDAP-2836 `__ - Added support for restart of specific CDAP System Services Instances. -- `CDAP-2853 `__ - +- `CDAP-2853 `__ - Completed certification process for MapR on CDAP. -- `CDAP-2879 `__ - +- `CDAP-2879 `__ - Added Hive 1.0 in Standalone. -- `CDAP-2881 `__ - +- `CDAP-2881 `__ - Added support for HDP 2.2.x. -- `CDAP-2891 `__ - +- `CDAP-2891 `__ - Documented cdap-env.sh and settings OPTS for HDP 2.2. -- `CDAP-2898 `__ - +- `CDAP-2898 `__ - Added Hive 1.1 in Standalone. -- `CDAP-2953 `__ - +- `CDAP-2953 `__ - Added HiveServer2 support in a secure cluster. **Spark** -- `CDAP-344 `__ - +- `CDAP-344 `__ - Users can now run Spark in distributed mode. -- `CDAP-1993 `__ - +- `CDAP-1993 `__ - Added ability to manipulate the SparkConf. -- `CDAP-2700 `__ - +- `CDAP-2700 `__ - Added the ability to Spark programs of discovering CDAP services in distributed mode. -- `CDAP-2701 `__ - +- `CDAP-2701 `__ - Spark programs are able to collect Metrics in distributed mode. -- `CDAP-2703 `__ - +- `CDAP-2703 `__ - Users are able to collect/view logs from Spark programs in distributed mode. -- `CDAP-2705 `__ - +- `CDAP-2705 `__ - Added examples, guides and documentation for Spark in distributed mode. LogAnalysis application demonstrating parallel execution of the Spark and MapReduce programs using Workflows. -- `CDAP-2923 `__ - +- `CDAP-2923 `__ - Added support for the WorkflowToken in the Spark programs. -- `CDAP-2936 `__ - +- `CDAP-2936 `__ - Spark program can now specify resources usage for driver and executor process in distributed mode. **Workflows** -- `CDAP-1983 `__ - +- `CDAP-1983 `__ - Added example application for processing and analyzing Wikipedia data using Workflows. -- `CDAP-2709 `__ - +- `CDAP-2709 `__ - Added ability to add generic keys to the WorkflowToken. -- `CDAP-2712 `__ - +- `CDAP-2712 `__ - Added ability to update the WorkflowToken in MapReduce and Spark programs. -- `CDAP-2713 `__ - +- `CDAP-2713 `__ - Added ability to persist the WorkflowToken per run of the Workflow. -- `CDAP-2714 `__ - +- `CDAP-2714 `__ - Added ability to query the WorkflowToken for the past as well as currently running Workflow runs. -- `CDAP-2752 `__ - +- `CDAP-2752 `__ - Added ability for custom actions to access the CDAP datasets and services. -- `CDAP-2894 `__ - +- `CDAP-2894 `__ - Added an API to retreive the system properties (e.g. MapReduce counters in case of MapReduce program) from the WorkflowToken. -- `CDAP-2923 `__ - +- `CDAP-2923 `__ - Added support for the WorkflowToken in the Spark programs. -- `CDAP-2982 `__ - +- `CDAP-2982 `__ - Added verification that the Workflow contains all programs/custom actions with a unique name. **Datasets** -- `CDAP-347 `__ - +- `CDAP-347 `__ - User can use datasets in beforeSubmit and afterFinish. -- `CDAP-585 `__ - +- `CDAP-585 `__ - Changes to Spark program runner to use File dataset in Spark. Spark programs can now use file-based datasets. -- `CDAP-2734 `__ - +- `CDAP-2734 `__ - Added PartitionedFileSet support to setting/getting properties at the Partition level. -- `CDAP-2746 `__ - +- `CDAP-2746 `__ - PartitionedFileSets now record the creation time of each partition in the metadata. -- `CDAP-2747 `__ - +- `CDAP-2747 `__ - PartitionedFileSets now index the creation time of partitions to allow selection of partitions that were created after a given time. Introduced BatchPartitionConsumer as a way to incrementally consume new data in a PartitionedFileSet. -- `CDAP-2752 `__ - +- `CDAP-2752 `__ - Added ability for custom actions to access the CDAP datasets and services. -- `CDAP-2758 `__ - +- `CDAP-2758 `__ - FileSet now support existing HDFS locations. Treat base paths that start with "/" as absolute in the file system. An absolute base @@ -5909,805 +5909,805 @@ New Features FileSet is dropped or truncated, and it will not allow adding or deleting files or partitions. In other words, the FileSet is read-only. -- `CDAP-2784 `__ - +- `CDAP-2784 `__ - Added support to write to PartitionedFileSet Partition metadata from MapReduce. -- `CDAP-2822 `__ - +- `CDAP-2822 `__ - IndexedTable now supports scans on the indexed field. **Metrics** -- `CDAP-2975 `__ - +- `CDAP-2975 `__ - Added pre-split FactTables. -- `CDAP-2326 `__ - +- `CDAP-2326 `__ - Added better unit-test coverage for Cube dataset. -- `CDAP-1853 `__ - +- `CDAP-1853 `__ - Metrics processor scaling no longer needs a master services restart. -- `CDAP-2844 `__ - +- `CDAP-2844 `__ - MapReduce metrics collection no longer use counters, and instead report directly to Kafka. -- `CDAP-2701 `__ - +- `CDAP-2701 `__ - Spark programs are able to collect Metrics in distributed mode. -- `CDAP-2466 `__ - +- `CDAP-2466 `__ - Added CLI for metrics search and query. -- `CDAP-2236 `__ - +- `CDAP-2236 `__ - New CDAP UI switched over to using newer search/query APIs. -- `CDAP-1998 `__ - +- `CDAP-1998 `__ - Removed deprecated Context - Query param in Metrics v3 API. **Miscellaneous New Features** -- `CDAP-332 `__ - +- `CDAP-332 `__ - Added a Restful end-point for deleting Streams. -- `CDAP-1483 `__ - +- `CDAP-1483 `__ - QueueAdmin now uses Id.Namespace instead of simply String. -- `CDAP-1584 `__ - +- `CDAP-1584 `__ - CDAP CLI now shows the username in the CLI prompt. -- `CDAP-2139 `__ - +- `CDAP-2139 `__ - Removed a duplicate Table of Contents on the Documentation Search page. -- `CDAP-2515 `__ - +- `CDAP-2515 `__ - Added a metrics client for search and query by tags. -- `CDAP-2582 `__ - +- `CDAP-2582 `__ - Documented the licenses of the shipped CDAP UI components. -- `CDAP-2595 `__ - +- `CDAP-2595 `__ - Added data modelling of flows. -- `CDAP-2596 `__ - +- `CDAP-2596 `__ - Added data modelling of MapReduce. -- `CDAP-2617 `__ - +- `CDAP-2617 `__ - Added the capability to get logs for a given time range from CLI. -- `CDAP-2618 `__ - +- `CDAP-2618 `__ - Simplified the Cube sink configurations. -- `CDAP-2670 `__ - +- `CDAP-2670 `__ - Added Parquet sink with time partitioned file dataset. -- `CDAP-2739 `__ - +- `CDAP-2739 `__ - Added S3 batch source for ETLbatch. -- `CDAP-2802 `__ - +- `CDAP-2802 `__ - Stopped using HiveConf.ConfVars.defaultValue, to support Hive >0.13. -- `CDAP-2847 `__ - +- `CDAP-2847 `__ - Added ability to add custom filters to FileBatchSource. -- `CDAP-2893 `__ - +- `CDAP-2893 `__ - Custom Transform now parses log formats for ETL. -- `CDAP-2913 `__ - +- `CDAP-2913 `__ - Provided installation method for EMR. -- `CDAP-2915 `__ - +- `CDAP-2915 `__ - Added an SQS real-time plugin for ETL. -- `CDAP-3022 `__ - +- `CDAP-3022 `__ - Added Cloudfront format option to LogParserTransform. -- `CDAP-3032 `__ - +- `CDAP-3032 `__ - Documented TestConfiguration class usage in unit-test framework. Improvements ------------ -- `CDAP-593 `__ - +- `CDAP-593 `__ - Spark no longer determines the mode through MRConfig.FRAMEWORK_NAME. -- `CDAP-595 `__ - +- `CDAP-595 `__ - Refactored SparkRuntimeService and SparkProgramWrapper. -- `CDAP-665 `__ - +- `CDAP-665 `__ - Documentation received a product-specifc 404 Page. -- `CDAP-683 `__ - +- `CDAP-683 `__ - Changed all README files from markdown to rst format. -- `CDAP-1132 `__ - +- `CDAP-1132 `__ - Improved the CDAP Doc Search Result Sorting. -- `CDAP-1416 `__ - +- `CDAP-1416 `__ - Added links to upper level pages on Docs. -- `CDAP-1572 `__ - +- `CDAP-1572 `__ - Standardized Id classes. -- `CDAP-1583 `__ - +- `CDAP-1583 `__ - Refactored InMemoryWorkerRunner and ServiceProgramRunnner after ServiceWorkers were removed. -- `CDAP-1918 `__ - +- `CDAP-1918 `__ - Switched to using the Spark 1.3.0 release. -- `CDAP-1926 `__ - +- `CDAP-1926 `__ - Streams endpoint accept "now", "now-30s", etc., for time ranges. -- `CDAP-2007 `__ - +- `CDAP-2007 `__ - CLI output for "call service" is rendered in a copy-pastable manner. -- `CDAP-2310 `__ - +- `CDAP-2310 `__ - Kafka Source now able to apply a Schema to the Payload received. -- `CDAP-2388 `__ - +- `CDAP-2388 `__ - Added Java 8 support to CDAP. -- `CDAP-2422 `__ - +- `CDAP-2422 `__ - Removed redundant catch blocks in AdapterHttpHandler. -- `CDAP-2455 `__ - +- `CDAP-2455 `__ - Version in CDAP UI footer is dynamic. -- `CDAP-2482 `__ - +- `CDAP-2482 `__ - Reduced excessive capitalisation in documentation. -- `CDAP-2531 `__ - +- `CDAP-2531 `__ - Adapter details made available through CDAP UI. -- `CDAP-2539 `__ - +- `CDAP-2539 `__ - Added a build identifier (branch, commit) in header of Documentation HTML pages. -- `CDAP-2552 `__ - +- `CDAP-2552 `__ - Documentation Build script now flags errors. -- `CDAP-2554 `__ - +- `CDAP-2554 `__ - Documented that streams can now be deleted. -- `CDAP-2557 `__ - +- `CDAP-2557 `__ - Non-handler logic moved out of DatasetInstanceHandler. -- `CDAP-2570 `__ - +- `CDAP-2570 `__ - CLI prompt changes to 'DISCONNECTED' after CDAP is stopped. -- `CDAP-2578 `__ - +- `CDAP-2578 `__ - Ability to look at configs of created adapters. -- `CDAP-2585 `__ - +- `CDAP-2585 `__ - Use Id in cdap-client rather than Id.Namespace + String. -- `CDAP-2588 `__ - +- `CDAP-2588 `__ - Improvements to the MetricsClient APIs. -- `CDAP-2590 `__ - +- `CDAP-2590 `__ - Switching namespaces when in CDAP UI Operations screens. -- `CDAP-2620 `__ - +- `CDAP-2620 `__ - CDAP clients now use Id classes from cdap proto, instead of plain strings. -- `CDAP-2628 `__ - +- `CDAP-2628 `__ - CDAP UI: Breadcrumbs in Workflow/Mapreduce work as expected. -- `CDAP-2644 `__ - +- `CDAP-2644 `__ - In cdap-clients, no longer need to retrieve runtime arguments before starting a program. -- `CDAP-2651 `__ - +- `CDAP-2651 `__ - CDAP UI: the Namespace is made more prominent. -- `CDAP-2681 `__ - +- `CDAP-2681 `__ - CDAP UI: scrolling no longer enlarges the workflow diagram instead of scrolling through. -- `CDAP-2683 `__ - +- `CDAP-2683 `__ - CDAP UI: added a remove icons for fork and Join. -- `CDAP-2684 `__ - +- `CDAP-2684 `__ - CDAP UI: workflow diagrams are directed graphs. -- `CDAP-2688 `__ - +- `CDAP-2688 `__ - CDAP UI: added search & pagination for lists of apps and datasets. -- `CDAP-2689 `__ - +- `CDAP-2689 `__ - CDAP UI: shows which application is a part of which dataset. -- `CDAP-2691 `__ - +- `CDAP-2691 `__ - CDAP UI: added ability to delete streams. -- `CDAP-2692 `__ - +- `CDAP-2692 `__ - CDAP UI: added pagination for logs. -- `CDAP-2694 `__ - +- `CDAP-2694 `__ - CDAP UI: added a loading icon/UI element when creating an adapter. -- `CDAP-2695 `__ - +- `CDAP-2695 `__ - CDAP UI: long names of adapters are replaced by a short version ending in an ellipsis. -- `CDAP-2697 `__ - +- `CDAP-2697 `__ - CDAP UI: added tab names during adapter creation. -- `CDAP-2716 `__ - +- `CDAP-2716 `__ - CDAP UI: when creating an adapter, the tabbing order shows correctly. -- `CDAP-2733 `__ - +- `CDAP-2733 `__ - Implemented a TimeParitionedFileSet source. -- `CDAP-2811 `__ - +- `CDAP-2811 `__ - Improved Hive version detection. -- `CDAP-2921 `__ - +- `CDAP-2921 `__ - Removed backward-compatibility for pre-2.8 TPFS. -- `CDAP-2938 `__ - +- `CDAP-2938 `__ - Implemented new ETL application template creation. -- `CDAP-2983 `__ - +- `CDAP-2983 `__ - Spark program runner now calls onFailure() of the DatasetOutputCommitter. -- `CDAP-2986 `__ - +- `CDAP-2986 `__ - Spark program now are able to specify runtime arguments when reading or writing a datset. -- `CDAP-2987 `__ - +- `CDAP-2987 `__ - Added an example for Spark using datasets directly. -- `CDAP-2989 `__ - +- `CDAP-2989 `__ - Added an example for Spark using FileSets. -- `CDAP-3018 `__ - +- `CDAP-3018 `__ - Updated workflow guides for workflow token. -- `CDAP-3028 `__ - +- `CDAP-3028 `__ - Improved the system service restart endpoint to handle illegal instance IDs and "service not available". -- `CDAP-3053 `__ - +- `CDAP-3053 `__ - Added schema javadocs that explain how to write the schema to JSON. -- `CDAP-3077 `__ - +- `CDAP-3077 `__ - Add the ability in TableSink to find schema.row.field case-insensitively. -- `CDAP-3144 `__ - +- `CDAP-3144 `__ - Changed CLI command descriptions to use consistent element case. -- `CDAP-3152 `__ - +- `CDAP-3152 `__ - Refactored ETLBatch sources and sinks. Bug Fixes --------- -- `CDAP-23 `__ - +- `CDAP-23 `__ - Fixed a problem with the DatasetFramework not loading a given dataset with the same classloader across calls. -- `CDAP-68 `__ - +- `CDAP-68 `__ - Made sure all network services in Singlenode only bind to localhost. -- `CDAP-376 `__ - +- `CDAP-376 `__ - Fixed a problem with HBaseOrderedTable never calling HTable.close(). -- `CDAP-550 `__ - +- `CDAP-550 `__ - Consolidated Examples, Guides, and Tutorials styles. -- `CDAP-598 `__ - +- `CDAP-598 `__ - Fixed problems with the CDAP ClassLoading model. -- `CDAP-674 `__ - +- `CDAP-674 `__ - Fixed problems with CDAP code examples and versioning. -- `CDAP-814 `__ - +- `CDAP-814 `__ - Resolved issues in the documentation about element versus program. -- `CDAP-1042 `__ - +- `CDAP-1042 `__ - Fixed a problem with specifying dataset selection as input for Spark job. -- `CDAP-1145 `__ - +- `CDAP-1145 `__ - Fixed the PurchaseAppTest. -- `CDAP-1184 `__ - +- `CDAP-1184 `__ - Fixed a problem with the DELETE call not clearing queue metrics. -- `CDAP-1273 `__ - +- `CDAP-1273 `__ - Fixed a problem with the ProgramClassLoader getResource. -- `CDAP-1457 `__ - +- `CDAP-1457 `__ - Fixed a memory leak of user class after running Spark program. -- `CDAP-1552 `__ - +- `CDAP-1552 `__ - Fixed a problem with Mapreduce progress metrics not being interpolated. -- `CDAP-1868 `__ - +- `CDAP-1868 `__ - Fixed a problem with Java Client and CLI not setting set dataset properties on existing datasets. -- `CDAP-1873 `__ - +- `CDAP-1873 `__ - Fixed a problem with warnings and errors when CDAP-Master starts up. -- `CDAP-1967 `__ - +- `CDAP-1967 `__ - Fixed a problem with CDAP-Master failing to start up due to conflicting dependencies. -- `CDAP-1976 `__ - +- `CDAP-1976 `__ - Fixed a problem with examples not following the same pattern. -- `CDAP-1988 `__ - +- `CDAP-1988 `__ - Fixed a problem with creating a Dataset through REST API failing if no properties are provided. -- `CDAP-2081 `__ - +- `CDAP-2081 `__ - Fixed a problem with StreamSizeSchedulerTest failing randomly. -- `CDAP-2140 `__ - +- `CDAP-2140 `__ - Fixed a problem with the CDAP UI not showing system service status when system services are down. -- `CDAP-2177 `__ - +- `CDAP-2177 `__ - Fixed a problem with Enable and Fix LogSaverPluginTest. -- `CDAP-2208 `__ - +- `CDAP-2208 `__ - Fixed a problem with CDAP-Explore service failing on wrapped indexedTable with Avro (specific record) contents. -- `CDAP-2228 `__ - +- `CDAP-2228 `__ - Fixed a problem with Mapreduce not working in Hadoop 2.2. -- `CDAP-2254 `__ - +- `CDAP-2254 `__ - Fixed a problem with an incorrect error message returned by HTTP RESTful Handler. -- `CDAP-2258 `__ - +- `CDAP-2258 `__ - Fixed a problem with an internal error when attempting to start a non-existing program. -- `CDAP-2279 `__ - +- `CDAP-2279 `__ - Fixed a problem with namespace and gear widgets disappearing when the browser window is too narrow. -- `CDAP-2280 `__ - +- `CDAP-2280 `__ - Fixed a problem when starting a flow from the GUI that the GUI does not fully refresh the page. -- `CDAP-2341 `__ - +- `CDAP-2341 `__ - Fixed a problem that when a MapReduce fails to start, it cannot be started or stopped any more. -- `CDAP-2343 `__ - +- `CDAP-2343 `__ - Fixed a problem in the CDAP UI that Mapreduce logs are convoluted with system logs. -- `CDAP-2344 `__ - +- `CDAP-2344 `__ - Fixed a problem with the formatting of logs in the CDAP UI. -- `CDAP-2355 `__ - +- `CDAP-2355 `__ - Fixed a problem with an Adapter CLI help error. -- `CDAP-2356 `__ - +- `CDAP-2356 `__ - Fixed a problem with CLI autocompletion results not sorted in alphabetical order. -- `CDAP-2365 `__ - +- `CDAP-2365 `__ - Fixed a problem that when restarting CDAP-Master, the CDAP UI oscillates between being up and down. -- `CDAP-2376 `__ - +- `CDAP-2376 `__ - Fixed a problem with logs from mapper and reducer not being collected. -- `CDAP-2444 `__ - +- `CDAP-2444 `__ - Fixed a problem with Cloudera Configuring doc needs fixing. -- `CDAP-2446 `__ - +- `CDAP-2446 `__ - Fixed a problem with that examples needing to be updated for new CDAP UI. -- `CDAP-2454 `__ - +- `CDAP-2454 `__ - Fixed a problem with Proto class RunRecord containing the Apache Twill RunId when serialized in REST API response. -- `CDAP-2459 `__ - +- `CDAP-2459 `__ - Fixed a problem with the CDAP UI going into a loop when the Router returns 200 and App Fabric is not up. -- `CDAP-2474 `__ - +- `CDAP-2474 `__ - Fixed a problem with changing the format of the name for the connectionfactory in JMS source plugin. -- `CDAP-2475 `__ - +- `CDAP-2475 `__ - Fixed a problem with JMS source accepting the type and name of the JMS provider plugin. -- `CDAP-2480 `__ - +- `CDAP-2480 `__ - Fixed a problem with the Workflow current run info endpoint missing a /runs/ in the path. -- `CDAP-2489 `__ - +- `CDAP-2489 `__ - Fixed a problem when, in distributed mode and CDAP master restarted, status of the running PROGRAM is always returned as STOPPED. -- `CDAP-2490 `__ - +- `CDAP-2490 `__ - Fixed a problem with checking if invalid Run Records for Spark and MapReduce are part of run from Workflow child programs. -- `CDAP-2491 `__ - +- `CDAP-2491 `__ - Fixed a problem with the MapReduce program in standalone mode not always using LocalJobRunnerWithFix. -- `CDAP-2493 `__ - - After the fix for `CDAP-2474 `__ (ConnectionFactory in JMS source), +- `CDAP-2493 `__ - + After the fix for `CDAP-2474 `__ (ConnectionFactory in JMS source), the JSON file requires updating for the change to reflect in CDAP UI. -- `CDAP-2496 `__ - +- `CDAP-2496 `__ - Fixed a problem with CDAP using its own transaction snapshot codec. -- `CDAP-2498 `__ - +- `CDAP-2498 `__ - Fixed a problem with validation while creating adapters only by types and not also by values. -- `CDAP-2517 `__ - +- `CDAP-2517 `__ - Fixed a problem with Explore docs not mentioning partitioned file sets. -- `CDAP-2520 `__ - +- `CDAP-2520 `__ - Fixed a problem with StreamSource not liking values of '0m'. -- `CDAP-2522 `__ - +- `CDAP-2522 `__ - Fixed a problem with TransactionStateCache needing to reference Tephra SnapshotCodecV3. -- `CDAP-2529 `__ - +- `CDAP-2529 `__ - Fixed a problem with CLI not printing an error if it can't connect to CDAP. -- `CDAP-2530 `__ - +- `CDAP-2530 `__ - Fixed a problem with Custom RecordScannable datasets not be explorable. -- `CDAP-2535 `__ - +- `CDAP-2535 `__ - Fixed a problem with IntegrationTestManager deployApplication not being namespaced. -- `CDAP-2538 `__ - +- `CDAP-2538 `__ - Fixed a problem with handling extra whitespace in CLI input. -- `CDAP-2540 `__ - +- `CDAP-2540 `__ - Fixed a problem with the Preferences Namespace CLI help having errors. -- `CDAP-2541 `__ - +- `CDAP-2541 `__ - Added the ability to stop the particular run of a program. Allows concurrent runs of the MapReduce and Workflow programs and the ability to stop programs at a per-run level. -- `CDAP-2547 `__ - +- `CDAP-2547 `__ - Fixed a problem with Kakfa Source - not using the persisted offset when the Adapter is restarted. -- `CDAP-2549 `__ - +- `CDAP-2549 `__ - Fixed a problem with a suspended workflow run record not being removed upon app/namespace delete. -- `CDAP-2562 `__ - +- `CDAP-2562 `__ - Fixed a problem with the automated Doc Build failing in develop. -- `CDAP-2564 `__ - +- `CDAP-2564 `__ - Improved the management of dataset resources. -- `CDAP-2565 `__ - +- `CDAP-2565 `__ - Fixed a problem with the transaction latency metric being of incorrect type. -- `CDAP-2569 `__ - +- `CDAP-2569 `__ - Fixed a problem with master process not being resilient to zookeeper exceptions. -- `CDAP-2571 `__ - +- `CDAP-2571 `__ - Fixed a problem with the RunRecord thread not resilient to errors. -- `CDAP-2587 `__ - +- `CDAP-2587 `__ - Fixed a problem with being unable to create default namespaces on starting up SDK. -- `CDAP-2635 `__ - +- `CDAP-2635 `__ - Fixed a problem with Namespace Create ignoring the properties' config field. -- `CDAP-2636 `__ - +- `CDAP-2636 `__ - Fixed a problem with "out of perm gen" space in CDAP Explore service. -- `CDAP-2654 `__ - +- `CDAP-2654 `__ - Fixed a problem with False values showing up as 'false null' in the CDAP Explore UI. -- `CDAP-2685 `__ - +- `CDAP-2685 `__ - Fixed a problem with the CDAP UI: no empty box for transforms. -- `CDAP-2729 `__ - +- `CDAP-2729 `__ - Fixed a problem with CDAP UI not handling downstream system services gracefully. -- `CDAP-2740 `__ - +- `CDAP-2740 `__ - Fixed a problem with CDAP UI not gracefully handling when the nodejs server goes down. -- `CDAP-2748 `__ - +- `CDAP-2748 `__ - Fixed a problem with the currently running and completed status of Spark programs in a workflow not highlighted in the UI. -- `CDAP-2765 `__ - +- `CDAP-2765 `__ - Fixed a problem with security warnings when CLI starts up. -- `CDAP-2766 `__ - +- `CDAP-2766 `__ - Fixed a problem with CLI asking for the user/password twice. -- `CDAP-2767 `__ - +- `CDAP-2767 `__ - Fixed a problem with incorrect error messages for namespace deletion. -- `CDAP-2768 `__ - +- `CDAP-2768 `__ - Fixed a problem with CLI and UI listing system.queue as a dataset. -- `CDAP-2769 `__ - +- `CDAP-2769 `__ - Fixed a problem with Use io.cdap.cdap.common.app.RunIds instead of org.apache.twill.internal.RunIds for InMemoryServiceProgramRunner. -- `CDAP-2787 `__ - +- `CDAP-2787 `__ - Fixed a problem when the number of MapReduce task metrics going over limit and causing MapReduce to fail. -- `CDAP-2796 `__ - +- `CDAP-2796 `__ - Fixed a problem with emitting duplicate metrics for dataset ops. -- `CDAP-2803 `__ - +- `CDAP-2803 `__ - Fixed a problem with scan operations not reflecting in dataset ops metrics. -- `CDAP-2804 `__ - +- `CDAP-2804 `__ - Fixed a problem with DataSetRecordReader incorrectly reporting dataset ops metrics. -- `CDAP-2810 `__ - +- `CDAP-2810 `__ - Fixed a problem with IncrementAndGet, CompareAndSwap, and Delete ops on Table incorrectly reporting two writes each. -- `CDAP-2821 `__ - +- `CDAP-2821 `__ - Fixed a problem with a Spark native library linkage error causing Standalone CDAP to stop. -- `CDAP-2823 `__ - +- `CDAP-2823 `__ - Fixed a problem with the conversion from Avro and to Avro not taking into account nested records. -- `CDAP-2830 `__ - +- `CDAP-2830 `__ - Fixed a problem with CDAP UI dying when CDAP Master is killed. -- `CDAP-2832 `__ - +- `CDAP-2832 `__ - Fixed a problem where suspending a schedule takes a long time and the CDAP UI does not provide any indication. -- `CDAP-2838 `__ - +- `CDAP-2838 `__ - Fixed a problem with poor error message when there is a mistake in security configration. -- `CDAP-2839 `__ - +- `CDAP-2839 `__ - Fixed a problem with the CDAP start script needing updating for the correct Node.js version. -- `CDAP-2848 `__ - +- `CDAP-2848 `__ - Fixed a problem with the Preferences Client test. -- `CDAP-2849 `__ - +- `CDAP-2849 `__ - Fixed a problem with the FileBatchSource reading files in twice if it takes longer that one workflow cycle to complete the job. -- `CDAP-2851 `__ - +- `CDAP-2851 `__ - Fixed a problem with RPM and DEB release artifacts being uploaded to incorrect staging directory. -- `CDAP-2854 `__ - +- `CDAP-2854 `__ - Fixed a problem with the instructions for using Docker. -- `CDAP-2855 `__ - +- `CDAP-2855 `__ - Fixed a problem with the example builds in VM failing with a Maven dependency error. -- `CDAP-2860 `__ - +- `CDAP-2860 `__ - Fixed a problem with the documentation for updating dataset properties. -- `CDAP-2861 `__ - +- `CDAP-2861 `__ - Fixed a problem with CDAP UI not mentioning required fields in all entry forms. -- `CDAP-2862 `__ - +- `CDAP-2862 `__ - Fixed a problem with CDAP UI creating multiple namespaces with the same name. -- `CDAP-2866 `__ - +- `CDAP-2866 `__ - Fixed a problem with FileBatchSource not reattempting to read in files if there is a failure. -- `CDAP-2870 `__ - +- `CDAP-2870 `__ - Fixed a problem with Workflow Diagrams. -- `CDAP-2871 `__ - +- `CDAP-2871 `__ - Fixed a problem with the Cloudera Manager Hbase Gateway dependency. -- `CDAP-2895 `__ - +- `CDAP-2895 `__ - Fixed a problem with a put operation on the WorkflowToken not throwing an exception. -- `CDAP-2899 `__ - +- `CDAP-2899 `__ - Fixed a problem with Mapreduce local dirs not getting cleaned up. -- `CDAP-2900 `__ - +- `CDAP-2900 `__ - Fixed a problem with exposing app.template.dir as a config option. -- `CDAP-2904 `__ - +- `CDAP-2904 `__ - Fixed a problem with "Make Request" button overlapping with paths when a path is long. -- `CDAP-2912 `__ - +- `CDAP-2912 `__ - Fixed a problem with HBaseQueueDebugger not sorting queue barriers correctly. -- `CDAP-2922 `__ - +- `CDAP-2922 `__ - Fixed a problem with datasets created through DynamicDatasetContext not having metrics context. Datasets in MapReduce and Spark programs, and workers, were not emitting metrics. -- `CDAP-2925 `__ - +- `CDAP-2925 `__ - Fixed a problem with the documentation on how to create datasets with properties. -- `CDAP-2932 `__ - +- `CDAP-2932 `__ - Fixed a problem with the AdapterClient getRuns method constructing a malformed URL. -- `CDAP-2935 `__ - +- `CDAP-2935 `__ - Fixed a problem with the logs endpoint to retrieve the latest entry not working correctly. -- `CDAP-2940 `__ - +- `CDAP-2940 `__ - Fixed a problem with the test case ArtifactStoreTest#testConcurrentSnapshotWrite. -- `CDAP-2941 `__ - +- `CDAP-2941 `__ - Fixed a problem with the ScriptTransform failing to initialize. -- `CDAP-2942 `__ - +- `CDAP-2942 `__ - Fixed a problem with the CDAP UI namespace dropdown failing on standalone restart. -- `CDAP-2948 `__ - +- `CDAP-2948 `__ - Fixed a problem with creating Adapters. -- `CDAP-2952 `__ - +- `CDAP-2952 `__ - Fixed a problem with the plugin avro library not being accessible to MapReduce. -- `CDAP-2955 `__ - +- `CDAP-2955 `__ - Fixed a problem with a NoSuchMethodException when trying to explore Datasets/Stream. -- `CDAP-2971 `__ - +- `CDAP-2971 `__ - Fixed a problem with the dataset registration not registering datasets for applications upon deploy. -- `CDAP-2972 `__ - +- `CDAP-2972 `__ - Fixed a problem with being unable to instantiate dataset in ETLWorker initialization. -- `CDAP-2981 `__ - +- `CDAP-2981 `__ - Fixed a problem with undoing a FileSets upgrade in favor of versioning and backward-compatibility. -- `CDAP-2991 `__ - +- `CDAP-2991 `__ - Fixed a problem with Explore not working when it launches a MapReduce job. -- `CDAP-2992 `__ - +- `CDAP-2992 `__ - Fixed a problem with CLI broken for secure CDAP. -- `CDAP-2996 `__ - +- `CDAP-2996 `__ - Fixed a problem with CDAP UI: Stop Run and Suspend Run buttons needed styling updates. -- `CDAP-2997 `__ - +- `CDAP-2997 `__ - Fixed a problem with SparkProgramRunnerTest failing randomly. -- `CDAP-2999 `__ - +- `CDAP-2999 `__ - Fixed a problem with MapReduce jobs showing the duration for tasks as 17 days before the mapper starts. -- `CDAP-3001 `__ - +- `CDAP-3001 `__ - Fixed a problem with truncating a custom dataset failing with internal server error. -- `CDAP-3002 `__ - +- `CDAP-3002 `__ - Fixed a problem with tick initialDelay not working properly. -- `CDAP-3003 `__ - +- `CDAP-3003 `__ - Fixed a problem with user metrics emitted from flowlets not being queryable using the flow's tags. -- `CDAP-3006 `__ - +- `CDAP-3006 `__ - Fixed a problem with updating cdap-spark-* archetypes. -- `CDAP-3007 `__ - +- `CDAP-3007 `__ - Fixed a problem with testing all Spark apps/guides to work with 3.1 (in dist mode). -- `CDAP-3009 `__ - +- `CDAP-3009 `__ - Fixed a problem with the stream conversion upgrade being in the upgrade tool in 3.1. -- `CDAP-3010 `__ - +- `CDAP-3010 `__ - Fixed a problem with a Bower Dependency Error. -- `CDAP-3011 `__ - +- `CDAP-3011 `__ - Fixed a problem with the IncrementSummingScannerTest failing intermittently. -- `CDAP-3012 `__ - +- `CDAP-3012 `__ - Fixed a problem with the DistributedWorkflowProgramRunner localizing the spark-assembly.jar even if the workflow does not contain a Spark program. -- `CDAP-3013 `__ - +- `CDAP-3013 `__ - Fixed a problem with excluding a Spark assembly jar when building a MapReduce job jar. -- `CDAP-3019 `__ - +- `CDAP-3019 `__ - Fixed a problem with the PartitionedFileSet dropPartition not deleting files under the partition. -- `CDAP-3021 `__ - +- `CDAP-3021 `__ - Fixed a problem with allowing Cloudfront data to use BatchFileFilter. -- `CDAP-3023 `__ - +- `CDAP-3023 `__ - Fixed a problem with flowlet instance count defaulting to 1. -- `CDAP-3024 `__ - +- `CDAP-3024 `__ - Fixed a problem with surfacing more logs in CDAP UI for System Services. -- `CDAP-3026 `__ - +- `CDAP-3026 `__ - Fixed a problem with updating SparkPageRank example docs. -- `CDAP-3027 `__ - +- `CDAP-3027 `__ - Fixed a problem with the DFSStreamHeartbeatsTest failing on clusters. -- `CDAP-3030 `__ - +- `CDAP-3030 `__ - Fixed a problem with the loading of custom datasets being broken after upgrading. -- `CDAP-3031 `__ - +- `CDAP-3031 `__ - Fixed a problem with deploying an app with a dataset with an invalid base path returning an "internal error". -- `CDAP-3037 `__ - +- `CDAP-3037 `__ - Fixed a problem with not being able to use a PartitionedFileSet in a custom dataset. If a custom dataset embedded a Table and a PartitionedFileSet, loading the dataset at runtime would fail. -- `CDAP-3038 `__ - +- `CDAP-3038 `__ - Fixed a problem with logs not showing up in UI when using Spark. -- `CDAP-3039 `__ - +- `CDAP-3039 `__ - Fixed a problem with worker not stopping at the end of a run method in standalone. -- `CDAP-3040 `__ - +- `CDAP-3040 `__ - Fixed a problem with flowlet and stream metrics not being available in distributed mode. -- `CDAP-3042 `__ - +- `CDAP-3042 `__ - Fixed a problem with the BufferingTable not merging buffered writes with multi-get results. -- `CDAP-3043 `__ - +- `CDAP-3043 `__ - Fixed a problem with the Javadocs being broken. -- `CDAP-3044 `__ - +- `CDAP-3044 `__ - Fixed a problem with the user service 'methods' field in service specifications being inaccurate. -- `CDAP-3058 `__ - +- `CDAP-3058 `__ - Fixed a problem with the NamespacedLocationFactory not appending correctly. -- `CDAP-3066 `__ - +- `CDAP-3066 `__ - Fixed a problem with FileBatchSource not failing properly. -- `CDAP-3067 `__ - +- `CDAP-3067 `__ - Fixed a problem with the UpgradeTool throwing a NullPointerException during UsageRegistry.upgrade(). -- `CDAP-3070 `__ - +- `CDAP-3070 `__ - Fixed a problem on Ubuntu 14.10 where removing JSON files from templates/plugins/ETLBatch breaks adapters. -- `CDAP-3072 `__ - +- `CDAP-3072 `__ - Fixed a problem with a documentation JavaScript bug. -- `CDAP-3073 `__ - +- `CDAP-3073 `__ - Fixed a problem with out-of-memory perm gen space. -- `CDAP-3085 `__ - +- `CDAP-3085 `__ - Fixed a problem with adding integration tests for datasets. -- `CDAP-3086 `__ - +- `CDAP-3086 `__ - Fixed a problem with the CDAP UI current adapter UI. -- `CDAP-3087 `__ - +- `CDAP-3087 `__ - Fixed a problem with CDAP UI: a session timeout on secure mode. -- `CDAP-3088 `__ - +- `CDAP-3088 `__ - Fixed a problem with CDAP UI: application types need to be updated. -- `CDAP-3092 `__ - +- `CDAP-3092 `__ - Fixed a problem with reading multiple files with one mapper in FileBatchSource. -- `CDAP-3096 `__ - +- `CDAP-3096 `__ - Fixed a problem with running MapReduce on HDP 2.2. -- `CDAP-3098 `__ - +- `CDAP-3098 `__ - Fixed problems with the CDAP UI Adapter UI. -- `CDAP-3099 `__ - +- `CDAP-3099 `__ - Fixed a problem with CDAP UI and that settings icons shift 2px when you click on them. -- `CDAP-3104 `__ - +- `CDAP-3104 `__ - Fixed a problem with CDAP Explore throwing an exception if a Table dataset does not set schema. -- `CDAP-3105 `__ - +- `CDAP-3105 `__ - Fixed a problem with LogParserTransform needing to emit HTTP status code info. -- `CDAP-3106 `__ - +- `CDAP-3106 `__ - Fixed a problem with Hive query - local MapReduce task failure on CDH-5.4. -- `CDAP-3125 `__ - +- `CDAP-3125 `__ - Fixed a problem with the WorkerProgramRunnerTest failing intermittently. -- `CDAP-3127 `__ - +- `CDAP-3127 `__ - Fixed a problem with the Kafka guide not working with CDAP 3.1.0. -- `CDAP-3132 `__ - +- `CDAP-3132 `__ - Fixed a problem with the ProgramLifecycleHttpHandlerTest failing intermittently. -- `CDAP-3145 `__ - +- `CDAP-3145 `__ - Fixed a problem with the Metrics processor not processing metrics. -- `CDAP-3146 `__ - +- `CDAP-3146 `__ - Fixed a problem with the CDAP VM build failing to instal the Eclipse plugin. -- `CDAP-3148 `__ - +- `CDAP-3148 `__ - Fixed a problem with CDAP Explore MapReduce queries failing due to MR-framework being localized in the mapper container. -- `CDAP-3149 `__ - +- `CDAP-3149 `__ - Fixed a problem with cycles in an adapter create page causing the browser to freeze. -- `CDAP-3151 `__ - +- `CDAP-3151 `__ - Fixed a problem with CDAP examples shipped with SDK using JDK 1.6. -- `CDAP-3161 `__ - +- `CDAP-3161 `__ - Fixed a problem with MapReduce no longer working with default Cloudera manager settings. -- `CDAP-3173 `__ - +- `CDAP-3173 `__ - Fixed a problem with upgrading to 3.1.0 crashing the HBase co-processor. -- `CDAP-3174 `__ - +- `CDAP-3174 `__ - Fixed a problem with the ETL source/transform/sinks descriptions and documentation. -- `CDAP-3175 `__ - +- `CDAP-3175 `__ - Fixed a problem with the AbstractFlowlet constructors being deprecated when they should not be. @@ -6743,67 +6743,67 @@ Known Issues `__. Instead, use the v2 API. It will be supported in a future release. -- `CDAP-797 `__ - +- `CDAP-797 `__ - When running secure Hadoop clusters, metrics and debug logs from MapReduce programs are not available. -- `CDAP-1007 `__ - +- `CDAP-1007 `__ - If the Hive Metastore is restarted while the CDAP Explore Service is running, the Explore Service remains alive, but becomes unusable. To correct, restart the CDAP Master |---| which will restart all services |---| as described under "Starting CDAP Services" for your particular Hadoop distribution in the `Installation documentation `__. -- `CDAP-1587 `__ - +- `CDAP-1587 `__ - CDAP internally creates tables in the "user" space that begin with the word ``"system"``. User datasets with names starting with ``"system"`` can conflict if they were to match one of those names. To avoid this, do not start any datasets with the word ``"system"``. -- `CDAP-1864 `__ - +- `CDAP-1864 `__ - Creating a dataset in a non-existent namespace manifests in the RESTful API with an incorrect error message. -- `CDAP-2632 `__ - +- `CDAP-2632 `__ - The application in the `cdap-kafka-ingest-guide `__ does not run on Ubuntu 14.x as of CDAP 3.0.x. -- `CDAP-2785 `__ - +- `CDAP-2785 `__ - In the CDAP UI, many buttons will remain in focus after being clicked, even if they should not retain focus. -- `CDAP-2831 `__ - +- `CDAP-2831 `__ - A workflow that is scheduled by time will not be run between the failure of the primary master and the time that the secondary takes over. This scheduled run will not be triggered at all. -- `CDAP-2878 `__ - +- `CDAP-2878 `__ - The semantics for TTL are confusing, in that the Table TTL property is interpreted as milliseconds in some contexts: ``DatasetDefinition.confgure()`` and ``getAdmin()``. -- `CDAP-2945 `__ - +- `CDAP-2945 `__ - If the input partition filter for a PartitionedFileSet does not match any partitions, MapReduce jobs can fail. -- `CDAP-3000 `__ - +- `CDAP-3000 `__ - The Workflow token is in an inconsistent state for nodes in a fork while the nodes of the fork are still running. It becomes consistent after the join. -- `CDAP-3101 `__ - +- `CDAP-3101 `__ - If there are more than 30 concurrent runs of a workflow, the runs will not be scheduled due to a Quartz exception. -- `CDAP-3179 `__ - +- `CDAP-3179 `__ - If you are using CDH 5.3 (CDAP 3.0.0) and are upgrading to CDH 5.4 (CDAP 3.1.0), you must first upgrade the underlying HBase before you upgrade CDAP. This means that you perform the CDH upgrade before upgrading the CDAP. -- `CDAP-3189 `__ - +- `CDAP-3189 `__ - Large MapReduce jobs can cause excessive logging in the CDAP logs. -- `CDAP-3221 `__ - +- `CDAP-3221 `__ - When running in Standalone CDAP, if a MapReduce job fails repeatedly, then the SDK hits an out-of-memory exception due to ``perm gen``. The Standalone needs restarting at this point. @@ -6816,7 +6816,7 @@ Bug Fix ------- - Fixed a Bower dependency error in the CDAP UI - (`CDAP-3010 `__). + (`CDAP-3010 `__). Known Issues ------------ @@ -6829,8 +6829,8 @@ Bug Fixes --------- - Fixed problems with the dataset upgrade tool - (`CDAP-2962 `__, - `CDAP-2897 `__). + (`CDAP-2962 `__, + `CDAP-2897 `__). Known Issues ------------ @@ -6845,7 +6845,7 @@ New Features - In the CDAP UI, mandatory parameters for Application Template creation are marked with asterisks, and if a user tries to create a template without one of those parameters, the missing parameter is highlighted - (`CDAP-2499 `__). + (`CDAP-2499 `__). Improvements @@ -6856,41 +6856,41 @@ Improvements - Added a tool (`HBaseQueueDebugger `__) that counts consumed and unconsumed entries in a flowlet queue - (`CDAP-2105 `__). + (`CDAP-2105 `__). **CDAP UI** - The currently executing node of a workflow is now highlighted in the CDAP UI - (`CDAP-2615 `__). + (`CDAP-2615 `__). - The list of datasets and the run histories in the CDAP UI are now paginated - (`CDAP-2626 `__, - `CDAP-2627 `__). + (`CDAP-2626 `__, + `CDAP-2627 `__). - Added improvements to the CDAP UI when creating Application Templates - (`CDAP-2601 `__, - `CDAP-2602 `__, - `CDAP-2603 `__, - `CDAP-2605 `__, - `CDAP-2606 `__, - `CDAP-2607 `__, - `CDAP-2610 `__). + (`CDAP-2601 `__, + `CDAP-2602 `__, + `CDAP-2603 `__, + `CDAP-2605 `__, + `CDAP-2606 `__, + `CDAP-2607 `__, + `CDAP-2610 `__). - Improved the error messages returned when there are problems creating Application Templates in the CDAP UI - (`CDAP-2597 `__). + (`CDAP-2597 `__). **CDAP SDK VM** - Added the Apache Flume agent flume-ng to the CDAP SDK VM - (`CDAP-2612 `__). + (`CDAP-2612 `__). - Added the ability to copy and paste to the CDAP SDK VM - (`CDAP-2611 `__). + (`CDAP-2611 `__). - Pre-downloaded the example dependencies into the CDAP SDK VM to speed building of the CDAP examples - (`CDAP-2613 `__). + (`CDAP-2613 `__). Bug Fixes @@ -6900,187 +6900,187 @@ Bug Fixes - Fixed a problem with the HBase store and flows with multiple queues, where one queue name is a prefix of another queue name - (`CDAP-1996 `__). + (`CDAP-1996 `__). - Fixed a problem with namespaces with underscores in the name crashing the Hadoop HBase region servers - (`CDAP-2110 `__). + (`CDAP-2110 `__). - Removed the requirement to specify the JDBC driver class property twice in the adaptor configuration for Database Sources and Sinks - (`CDAP-2453 `__). + (`CDAP-2453 `__). - Fixed a problem in Distributed CDAP where the status of running program always returns as "STOPPED" when the CDAP Master is restarted - (`CDAP-2489 `__). + (`CDAP-2489 `__). - Fixed a problem with invalid RunRecords for Spark and MapReduce programs that are run as - part of a Workflow (`CDAP-2490 `__). + part of a Workflow (`CDAP-2490 `__). - Fixed a problem with the CDAP Master not being HA (highly available) when a leadership change happens - (`CDAP-2495 `__). + (`CDAP-2495 `__). - Fixed a problem with upgrading of queues with the UpgradeTool - (`CDAP-2502 `__). + (`CDAP-2502 `__). - Fixed a problem with ObjectMappedTables not deleting missing fields when updating a row - (`CDAP-2523 `__, - `CDAP-2524 `__). + (`CDAP-2523 `__, + `CDAP-2524 `__). - Fixed a problem with a stream not being created properly when deploying an application after the default namespace was deleted - (`CDAP-2537 `__). + (`CDAP-2537 `__). - Fixed a problem with the Applicaton Template Kafka Source not using the persisted offset when the Adapter is restarted - (`CDAP-2547 `__). + (`CDAP-2547 `__). - A problem with CDAP using its own transaction snapshot codec, leading to huge snapshot files and OutOfMemory exceptions, and transaction snapshots that can't be read using Tephra's tools, has been resolved by replacing the codec with Tephra's SnapshotCodecV3 - (`CDAP-2563 `__, - `CDAP-2946 `__, - `TEPHRA-101 `__). + (`CDAP-2563 `__, + `CDAP-2946 `__, + `TEPHRA-101 `__). - Fixed a problem with CDAP Master not being resilient in the handling of ZooKeeper exceptions - (`CDAP-2569 `__). + (`CDAP-2569 `__). - Fixed a problem with RunRecords not being cleaned up correctly after certain exceptions - (`CDAP-2584 `__). + (`CDAP-2584 `__). - Fixed a problem with the CDAP Maven archetype having an incorrect CDAP version in it - (`CDAP-2634 `__). + (`CDAP-2634 `__). - Fixed a problem with the description of the TwitterSource not describing its output - (`CDAP-2648 `__). + (`CDAP-2648 `__). - Fixed a problem with the Twitter Source not handling missing fields correctly and as a consequence producing tweets (with errors) that were then not stored on disk - (`CDAP-2653 `__). + (`CDAP-2653 `__). - Fixed a problem with the TwitterSource not calculating the time of tweet correctly - (`CDAP-2656 `__). + (`CDAP-2656 `__). - Fixed a problem with the JMS Real-time Source failing to load required plugin sources - (`CDAP-2661 `__). + (`CDAP-2661 `__). - Fixed a problem with executing Hive queries on a distributed CDAP due to a failure to load Grok classes - (`CDAP-2678 `__). + (`CDAP-2678 `__). - Fixed a problem with CDAP Program jars not being cleaned up from the temporary directory - (`CDAP-2698 `__). + (`CDAP-2698 `__). - Fixed a problem with ProjectionTransforms not handling input data fields with null values correctly - (`CDAP-2719 `__). + (`CDAP-2719 `__). - Fixed a problem with the CDAP SDK running out of memory when MapReduce jobs are run repeatedly - (`CDAP-2743 `__). + (`CDAP-2743 `__). - Fixed a problem with not using CDAP RunIDs in the in-memory version of the CDAP SDK - (`CDAP-2769 `__). + (`CDAP-2769 `__). **CDAP CLI** - Fixed a problem with the CDAP CLI not printing an error if it is unable to connect to a CDAP instance - (`CDAP-2529 `__). + (`CDAP-2529 `__). - Fixed a problem with extra whitespace in commands entered into the CDAP CLI causing errors - (`CDAP-2538 `__). + (`CDAP-2538 `__). **CDAP SDK Standalone** - Updated the messages displayed when starting the Standalone CDAP SDK as to components - and the JVM required (`CDAP-2445 `__). + and the JVM required (`CDAP-2445 `__). - Fixed a problem with the creation of the default namespace upon starting the CDAP SDK - (`CDAP-2587 `__). + (`CDAP-2587 `__). **CDAP SDK VM** - Fixed a problem with using the default namespace on the CDAP SDK Virtual Machine Image - (`CDAP-2500 `__). + (`CDAP-2500 `__). - Fixed a problem with the VirtualBox VM retaining a MAC address obtained from the build host - (`CDAP-2640 `__). + (`CDAP-2640 `__). **CDAP UI** - Fixed a problem with incorrect flow metrics showing in the CDAP UI - (`CDAP-2494 `__). + (`CDAP-2494 `__). - Fixed a problem in the CDAP UI with the properties in the Projection Transform being displayed inconsistently - (`CDAP-2525 `__). + (`CDAP-2525 `__). - Fixed a problem in the CDAP UI not automatically updating the number of flowlet instances - (`CDAP-2534 `__). + (`CDAP-2534 `__). - Fixed a problem in the CDAP UI with a window resize preventing clicking of the Adapter Template drop down menu - (`CDAP-2573 `__). + (`CDAP-2573 `__). - Fixed a problem with the CDAP UI not performing validation of mandatory parameters before the creation of an adapter - (`CDAP-2575 `__). + (`CDAP-2575 `__). - Fixed a problem with an incorrect version of CDAP being shown in the CDAP UI - (`CDAP-2586 `__). + (`CDAP-2586 `__). - Reduced the number of clicks required to navigate and perform actions within the CDAP UI - (`CDAP-2622 `__, - `CDAP-2625 `__). + (`CDAP-2622 `__, + `CDAP-2625 `__). - Fixed a problem with an additional forward-slash character in the URL causing a "page not found error" in the CDAP UI - (`CDAP-2624 `__). + (`CDAP-2624 `__). - Fixed a problem with the error dropdown of the CDAP UI not scrolling when it has a large number of errors - (`CDAP-2633 `__). + (`CDAP-2633 `__). - Fixed a problem in the CDAP UI with the Twitter Source's consumer key secret not being treated as a password field - (`CDAP-2649 `__). + (`CDAP-2649 `__). - Fixed a problem with the CDAP UI attempting to create an adapter without a name - (`CDAP-2652 `__). + (`CDAP-2652 `__). - Fixed a problem with the CDAP UI not being able to find the ETL plugin templates on distributed CDAP - (`CDAP-2655 `__). + (`CDAP-2655 `__). - Fixed a problem with the CDAP UI's System Dashboard chart having a y-axis starting at "-200" - (`CDAP-2699 `__). + (`CDAP-2699 `__). - Fixed a problem with the rendering of stack trace logs in the CDAP UI - (`CDAP-2745 `__). + (`CDAP-2745 `__). - Fixed a problem with the CDAP UI not working with secure CDAP instances, either clusters or standalone - (`CDAP-2770 `__). + (`CDAP-2770 `__). - Fixed a problem with the coloring of completed runs of Workflow DAGs in the CDAP UI - (`CDAP-2781 `__). + (`CDAP-2781 `__). **Documentation** - Fixed errors with the documentation examples of the ETL Plugins - (`CDAP-2503 `__). + (`CDAP-2503 `__). - Documented the licenses of all shipped CDAP UI components - (`CDAP-2582 `__). + (`CDAP-2582 `__). - Corrected issues with the building of Javadocs used on the website and removed Javadocs previously included in the SDK - (`CDAP-2730 `__). + (`CDAP-2730 `__). - Added a recommended version (v.12.0) of Node.js to the documentation - (`CDAP-2762 `__). + (`CDAP-2762 `__). .. _known-issues-301: @@ -7091,26 +7091,26 @@ Known Issues - The application in the `cdap-kafka-ingest-guide `__ does not run on Ubuntu 14.x and CDAP 3.0.x - (`CDAP-2632 `__, - `CDAP-2749 `__). + (`CDAP-2632 `__, + `CDAP-2749 `__). - Metrics for `TimePartitionedFileSets `__ can show zero values even if there is data present - (`CDAP-2721 `__). + (`CDAP-2721 `__). - In the CDAP UI: many buttons will remain in focus after being clicked, even if they should not retain focus - (`CDAP-2785 `__). + (`CDAP-2785 `__). - When the CDAP-Master dies, the CDAP UI does not repsond appropriately, and instead of waiting for routing to the secondary master to begin, it loses its connection - (`CDAP-2830 `__). + (`CDAP-2830 `__). - A workflow that is scheduled by time will not be run between the failure of the primary master and the time that the secondary takes over. This scheduled run will not be triggered at all. There is no warnings or messages about the missed run of the - workflow. (`CDAP-2831 `__) + workflow. (`CDAP-2831 `__) - CDAP has been tested on and supports CDH 4.2.x through CDH 5.3.x, HDP 2.0 through 2.1, and Apache Bigtop 0.8.0. It has not been tested on more recent versions of CDH. @@ -7124,19 +7124,19 @@ Known Issues - See the above section (*API Changes*) for alterations that can affect existing installations. - When running secure Hadoop clusters, metrics and debug logs from MapReduce programs are - not available (`CDAP-797 `__). + not available (`CDAP-797 `__). - If the Hive Metastore is restarted while the CDAP Explore Service is running, the Explore Service remains alive, but becomes unusable. To correct, `restart the CDAP Master `__, - which will restart all services (`CDAP-1007 `__). + which will restart all services (`CDAP-1007 `__). - User datasets with names starting with ``"system"`` can potentially cause conflicts - (`CDAP-1587 `__). + (`CDAP-1587 `__). - Scaling the number of metrics processor instances doesn't automatically distribute the processing load to the newer instances of the metrics processor. The CDAP Master needs to be restarted to effectively distribute the processing across all metrics processor instances - (`CDAP-1853 `__). + (`CDAP-1853 `__). - Creating a dataset in a non-existent namespace manifests in the RESTful API with an - incorrect error message (`CDAP-1864 `__). + incorrect error message (`CDAP-1864 `__). - Retrieving multiple metrics |---| by issuing an HTTP POST request with a JSON list as the request body that enumerates the name and attributes for each metric |---| is currently not supported in the @@ -7145,7 +7145,7 @@ Known Issues - Typically, datasets are bundled as part of applications. When an application is upgraded and redeployed, any changes in datasets will not be redeployed. This is because datasets can be shared across applications, and an incompatible schema change can break other applications that are using the dataset. - A workaround (`CDAP-1253 `__) is to allow *unchecked dataset upgrades*. + A workaround (`CDAP-1253 `__) is to allow *unchecked dataset upgrades*. Upgrades cause the dataset metadata, i.e. its specification including properties, to be updated. The dataset runtime code is also updated. To prevent data loss the existing data and the underlying HBase tables remain as-is. @@ -7165,9 +7165,9 @@ Known Issues New Features ------------ -- Support for application templates has been added (`CDAP-1753 `__). +- Support for application templates has been added (`CDAP-1753 `__). -- Built-in ETL application templates and plugins have been added (`CDAP-1767 `__). +- Built-in ETL application templates and plugins have been added (`CDAP-1767 `__). - New `CDAP UI `__, supports creating ETL applications directly in the web UI. @@ -7175,29 +7175,29 @@ New Features - Workflow logs can now be retrieved using the `CDP HTTP Logging RESTful API `__ - (`CDAP-1089 `__). + (`CDAP-1089 `__). - Support has been added for suspending and resuming of a workflow (`CDAP-1610 - `__). + `__). - Condition nodes in a workflow now allow branching based on a boolean predicate - (`CDAP-1928 `__). + (`CDAP-1928 `__). - Condition nodes in a workflow now allow passing the Hadoop counters from a MapReduce program to following Condition nodes in the workflow (`CDAP-1611 - `__). + `__). - Logs can now be fetched based on the ``run-id`` (`CDAP-1582 - `__). + `__). - CDAP Tables are `now explorable `__ - (`CDAP-946 `__). + (`CDAP-946 `__). - The `CDAP CLI `__ supports the new `application template and adapters APIs `__. - (`CDAP-1773 `__). + (`CDAP-1773 `__). - The `CDAP CLI `__ startup options have been changed to accommodate a new option @@ -7207,23 +7207,23 @@ New Features `syslog `__ record formats can now be used when `setting the format of a stream `__ - (`CDAP-1949 `__). + (`CDAP-1949 `__). - Added HTTP RESTful endpoints for listing datasets and streams as used by adapters, programs, and applications, and vice-versa - (`CDAP-2214 `__). + (`CDAP-2214 `__). - Created a `queue introspection tool `__, for counting processed and unprocessed entries in a - flowlet queue (`CDAP-2105 `__). + flowlet queue (`CDAP-2105 `__). -- Support for CDAP SDK VM build automation has been added (`CDAP-2030 `__). +- Support for CDAP SDK VM build automation has been added (`CDAP-2030 `__). -- A Cube dataset has been added (`CDAP-1520 `__). +- A Cube dataset has been added (`CDAP-1520 `__). -- A Batch and Real-Time Cube dataset sink has been added (`CDAP-1520 `__). +- A Batch and Real-Time Cube dataset sink has been added (`CDAP-1520 `__). -- Metrics and status information for MapReduce on a task level is now exposed (`CDAP-1520 `__). +- Metrics and status information for MapReduce on a task level is now exposed (`CDAP-1520 `__). .. _new-user-interface-300: @@ -7274,49 +7274,49 @@ Improvement - The `metrics system APIs `__ have been revised and improved - (`CDAP-1596 `__). + (`CDAP-1596 `__). - The metrics system performance has been improved - (`CDAP-2124 `__, - `CDAP-2125 `__). + (`CDAP-2124 `__, + `CDAP-2125 `__). Bug Fixes --------- - The CDAP Authentication server now reports the port correctly when the port is set to 0 - (`CDAP-614 `__). + (`CDAP-614 `__). - History of the programs running under workflow (Spark and MapReduce) is now updated correctly - (`CDAP-1293 `__). + (`CDAP-1293 `__). - Programs running under a workflow now receive a unique ``run-id`` - (`CDAP-2025 `__). + (`CDAP-2025 `__). - RunRecords are now updated with the RuntimeService to account for node failures - (`CDAP-2202 `__). + (`CDAP-2202 `__). - MapReduce metrics are now available on a secure cluster - (`CDAP-64 `__). + (`CDAP-64 `__). API Changes ----------- - The endpoint (``POST '/metrics/search?target=childContext[&context=]'``) that searched for the available contexts of metrics has been deprecated, pending removal - in a later version of CDAP (`CDAP-1998 `__). A + in a later version of CDAP (`CDAP-1998 `__). A `replacement endpoint `__ is available. - The endpoint (``POST '/metrics/search?target=metric&context='``) that searched for metrics in a specified context has been deprecated, pending removal - in a later version of CDAP (`CDAP-1998 `__). A + in a later version of CDAP (`CDAP-1998 `__). A `replacement endpoint `__ is available. - The endpoint (``POST '/metrics/query?context=[&groupBy=]&metric=&'``) that queried for a metric has been deprecated, pending removal - in a later version of CDAP (`CDAP-1998 `__). A + in a later version of CDAP (`CDAP-1998 `__). A `replacement endpoint `__ is available. @@ -7331,12 +7331,12 @@ API Changes startup options have been changed to accommodate a new option of executing a file containing a series of CLI commands, line-by-line. -- The metrics system APIs have been improved (`CDAP-1596 `__). +- The metrics system APIs have been improved (`CDAP-1596 `__). - The rules for `resolving resolution `__ when using ``resolution=auto`` in metrics query have been changed - (`CDAP-1922 `__). + (`CDAP-1922 `__). - Backward incompatible changes in ``InputFormatProvider`` and ``OutputFormatProvider``. It won't affect user code that uses ``FileSet`` or ``PartitionedFileSet``. @@ -7384,19 +7384,19 @@ Known Issues - See the above section (*API Changes*) for alterations that can affect existing installations. - When running secure Hadoop clusters, metrics and debug logs from MapReduce programs are - not available (`CDAP-797 `__). + not available (`CDAP-797 `__). - If the Hive Metastore is restarted while the CDAP Explore Service is running, the Explore Service remains alive, but becomes unusable. To correct, `restart the CDAP Master `__, - which will restart all services (`CDAP-1007 `__). + which will restart all services (`CDAP-1007 `__). - User datasets with names starting with ``"system"`` can potentially cause conflicts - (`CDAP-1587 `__). + (`CDAP-1587 `__). - Scaling the number of metrics processor instances doesn't automatically distribute the processing load to the newer instances of the metrics processor. The CDAP Master needs to be restarted to effectively distribute the processing across all metrics processor instances - (`CDAP-1853 `__). + (`CDAP-1853 `__). - Creating a dataset in a non-existent namespace manifests in the RESTful API with an - incorrect error message (`CDAP-1864 `__). + incorrect error message (`CDAP-1864 `__). - Retrieving multiple metrics |---| by issuing an HTTP POST request with a JSON list as the request body that enumerates the name and attributes for each metric |---| is currently not supported in the @@ -7405,7 +7405,7 @@ Known Issues - Typically, datasets are bundled as part of applications. When an application is upgraded and redeployed, any changes in datasets will not be redeployed. This is because datasets can be shared across applications, and an incompatible schema change can break other applications that are using the dataset. - A workaround (`CDAP-1253 `__) is to allow *unchecked dataset upgrades*. + A workaround (`CDAP-1253 `__) is to allow *unchecked dataset upgrades*. Upgrades cause the dataset metadata, i.e. its specification including properties, to be updated. The dataset runtime code is also updated. To prevent data loss the existing data and the underlying HBase tables remain as-is. @@ -7430,21 +7430,21 @@ General `__. - Added log rotation for CDAP programs running in YARN containers - (`CDAP-1295 `__). + (`CDAP-1295 `__). - Added the ability to submit to non-default YARN queues to provide `resource guarantees `__ for CDAP Master services, CDAP programs, and Explore Queries - (`CDAP-1417 `__). + (`CDAP-1417 `__). - Added the ability to `prune invalid transactions `__ - (`CDAP-1540 `__). + (`CDAP-1540 `__). - Added the ability to specify `custom logback file for CDAP programs `__ - (`CDAP-1100 `__). + (`CDAP-1100 `__). - System HTTP services now bind to all interfaces (0.0.0.0), rather than 127.0.0.1. @@ -7466,7 +7466,7 @@ New Features - Added a PartitionedFileSet dataset that allows addressing files by meta data and that is also explorable. - Table datasets now support a multi-get operation for batched reads. - Allow an unchecked dataset upgrade upon application deployment - (`CDAP-1574 `__). + (`CDAP-1574 `__). - **Metrics** @@ -7492,7 +7492,7 @@ New Features - **Spark** - Spark now uses a configurer-style API for specifying - (`CDAP-382 `__). + (`CDAP-382 `__). - **Workflows** @@ -7530,7 +7530,7 @@ API Changes - The endpoint (``GET /data/explore/datasets//schema``) that retrieved the schema of a dataset's underlying Hive table has been removed - (`CDAP-1603 `__). + (`CDAP-1603 `__). - Endpoints have been added to retrieve the CDAP version and the current configurations of CDAP and HBase (`Configuration HTTP RESTful API `__). @@ -7541,20 +7541,20 @@ API Changes Known Issues ------------ - When running secure Hadoop clusters, metrics and debug logs from MapReduce programs are - not available (`CDAP-64 `__ and `CDAP-797 - `__). + not available (`CDAP-64 `__ and `CDAP-797 + `__). - If the Hive Metastore is restarted while the CDAP Explore Service is running, the Explore Service remains alive, but becomes unusable. To correct, `restart the CDAP Master `__, - which will restart all services (`CDAP-1007 `__). + which will restart all services (`CDAP-1007 `__). - User datasets with names starting with ``"system"`` can potentially cause conflicts - (`CDAP-1587 `__). + (`CDAP-1587 `__). - Scaling the number of metrics processor instances doesn't automatically distribute the processing load to the newer instances of the metrics processor. The CDAP Master needs to be restarted to effectively distribute the processing across all metrics processor instances - (`CDAP-1853 `__). + (`CDAP-1853 `__). - Creating a dataset in a non-existent namespace manifests in the RESTful API with an - incorrect error message (`CDAP-1864 `__). + incorrect error message (`CDAP-1864 `__). - Retrieving multiple metrics |---| by issuing an HTTP POST request with a JSON list as the request body that enumerates the name and attributes for each metric |---| is currently not supported in the @@ -7563,7 +7563,7 @@ Known Issues - Typically, datasets are bundled as part of applications. When an application is upgraded and redeployed, any changes in datasets will not be redeployed. This is because datasets can be shared across applications, and an incompatible schema change can break other applications that are using the dataset. - A workaround (`CDAP-1253 `__) is to allow *unchecked dataset upgrades*. + A workaround (`CDAP-1253 `__) is to allow *unchecked dataset upgrades*. Upgrades cause the dataset metadata, i.e. its specification including properties, to be updated. The dataset runtime code is also updated. To prevent data loss the existing data and the underlying HBase tables remain as-is. @@ -7598,41 +7598,41 @@ Known Issues API Changes ----------- - The property ``security.auth.server.address`` has been deprecated and replaced with - ``security.auth.server.bind.address`` (`CDAP-639 `__, - `CDAP-1078 `__). + ``security.auth.server.bind.address`` (`CDAP-639 `__, + `CDAP-1078 `__). New Features ------------ - **Spark** - - Spark now uses a configurer-style API for specifying (`CDAP-382 `__). - - Spark can now run as a part of a workflow (`CDAP-465 `__). + - Spark now uses a configurer-style API for specifying (`CDAP-382 `__). + - Spark can now run as a part of a workflow (`CDAP-465 `__). - **Security** - - CDAP Master now obtains and refreshes Kerberos tickets programmatically (`CDAP-1134 `__). + - CDAP Master now obtains and refreshes Kerberos tickets programmatically (`CDAP-1134 `__). - **Datasets** - A new, experimental dataset type to support time-partitioned File sets has been added. - - Time-partitioned File sets can be queried with Impala on CDH distributions (`CDAP-926 `__). + - Time-partitioned File sets can be queried with Impala on CDH distributions (`CDAP-926 `__). - Streams can be made queryable with Impala by deploying an adapter that periodically - converts it into partitions of a time-partitioned File set (`CDAP-1129 `__). - - Support for different levels of conflict detection: ``ROW``, ``COLUMN``, or ``NONE`` (`CDAP-1016 `__). - - Removed support for ``@DisableTransaction`` (`CDAP-1279 `__). - - Support for annotating a stream with a schema (`CDAP-606 `__). - - A new API for uploading entire files to a stream has been added (`CDAP-411 `__). + converts it into partitions of a time-partitioned File set (`CDAP-1129 `__). + - Support for different levels of conflict detection: ``ROW``, ``COLUMN``, or ``NONE`` (`CDAP-1016 `__). + - Removed support for ``@DisableTransaction`` (`CDAP-1279 `__). + - Support for annotating a stream with a schema (`CDAP-606 `__). + - A new API for uploading entire files to a stream has been added (`CDAP-411 `__). - **Workflow** - - Workflow now uses a configurer-style API for specifying (`CDAP-1207 `__). - - Multiple instances of a workflow can be run concurrently (`CDAP-513 `__). + - Workflow now uses a configurer-style API for specifying (`CDAP-1207 `__). + - Multiple instances of a workflow can be run concurrently (`CDAP-513 `__). - Programs are no longer part of a workflow; instead, they are added in the application - and are referenced by a workflow using their names (`CDAP-1116 `__). + and are referenced by a workflow using their names (`CDAP-1116 `__). - Schedules are now at the application level and properties can be specified for Schedules; these properties will be passed to the scheduled program as runtime - arguments (`CDAP-1148 `__). + arguments (`CDAP-1148 `__). .. _known-issues-271: @@ -7640,13 +7640,13 @@ Known Issues ------------ - When upgrading an existing CDAP installation to 2.7.1, all metrics are reset. - When running secure Hadoop clusters, metrics and debug logs from MapReduce programs are - not available (`CDAP-64 `__ and `CDAP-797 - `__). + not available (`CDAP-64 `__ and `CDAP-797 + `__). - When upgrading a cluster from an earlier version of CDAP, warning messages may appear in the master log indicating that in-transit (emitted, but not yet processed) metrics system messages could not be decoded (*Failed to decode message to MetricsRecord*). This is because of a change in the format of emitted metrics, and can result in a small - amount of metrics data points being lost (`CDAP-745 `__). + amount of metrics data points being lost (`CDAP-745 `__). - A race condition resulting in a deadlock can occur when a TwillRunnable container shutdowns while it still has ZooKeeper events to process. This occasionally surfaces when running with OpenJDK or JDK7, though not with Oracle JDK6. It is caused by a change in the @@ -7666,7 +7666,7 @@ Known Issues - Typically, datasets are bundled as part of applications. When an application is upgraded and redeployed, any changes in datasets will not be redeployed. This is because datasets can be shared across applications, and an incompatible schema change can break other applications that are using the dataset. - A workaround (`CDAP-1253 `__) is to allow *unchecked dataset upgrades*. + A workaround (`CDAP-1253 `__) is to allow *unchecked dataset upgrades*. Upgrades cause the dataset metadata, i.e. its specification including properties, to be updated. The dataset runtime code is also updated. To prevent data loss the existing data and the underlying HBase tables remain as-is. @@ -7686,24 +7686,24 @@ Known Issues CDAP Bug Fixes -------------- - Allow an *unchecked dataset upgrade* upon application deployment - (`CDAP-1253 `__). + (`CDAP-1253 `__). - Update the Hive dataset table when a dataset is updated - (`CDAP-71 `__). + (`CDAP-71 `__). - Use Hadoop configuration files bundled with the Explore Service - (`CDAP-1250 `__). + (`CDAP-1250 `__). .. _known-issues-261: Known Issues ------------ - When running secure Hadoop clusters, metrics and debug logs from MapReduce programs are - not available (`CDAP-64 `__ and `CDAP-797 - `__). + not available (`CDAP-64 `__ and `CDAP-797 + `__). - When upgrading a cluster from an earlier version of CDAP, warning messages may appear in the master log indicating that in-transit (emitted, but not yet processed) metrics system messages could not be decoded (*Failed to decode message to MetricsRecord*). This is because of a change in the format of emitted metrics, and can result in a small - amount of metrics data points being lost (`CDAP-745 `__). + amount of metrics data points being lost (`CDAP-745 `__). - A race condition resulting in a deadlock can occur when a TwillRunnable container shutdowns while it still has ZooKeeper events to process. This occasionally surfaces when running with OpenJDK or JDK7, though not with Oracle JDK6. It is caused by a change in the @@ -7723,7 +7723,7 @@ Known Issues - Typically, datasets are bundled as part of applications. When an application is upgraded and redeployed, any changes in datasets will not be redeployed. This is because datasets can be shared across applications, and an incompatible schema change can break other applications that are using the dataset. - A workaround (`CDAP-1253 `__) is to allow *unchecked dataset upgrades*. + A workaround (`CDAP-1253 `__) is to allow *unchecked dataset upgrades*. Upgrades cause the dataset metadata, i.e. its specification including properties, to be updated. The dataset runtime code is also updated. To prevent data loss the existing data and the underlying HBase tables remain as-is. @@ -7745,7 +7745,7 @@ API Changes - API for specifying services and MapReduce programs has been changed to use a "configurer" style; this will require modification of user classes implementing either MapReduce or service as the interfaces have changed (`CDAP-335 - `__). + `__). New Features @@ -7754,83 +7754,83 @@ New Features - **General** - Health checks are now available for CDAP system services - (`CDAP-663 `__). + (`CDAP-663 `__). - **Applications** - Jar deployment now uses a chunked request and writes to a local temp file - (`CDAP-91 `__). + (`CDAP-91 `__). - **MapReduce** - MapReduce programs can now read binary stream data - (`CDAP-331 `__). + (`CDAP-331 `__). - **Datasets** - Added `FileSet `__, a new core dataset type for working with sets of files - (`CDAP-1 `__). + (`CDAP-1 `__). - **Spark** - Spark programs now emit system and custom user metrics - (`CDAP-346 `__). + (`CDAP-346 `__). - Services can be called from Spark programs and its worker nodes - (`CDAP-348 `__). + (`CDAP-348 `__). - Spark programs can now read from streams - (`CDAP-403 `__). + (`CDAP-403 `__). - Added Spark support to the CDAP CLI (Command-line Interface) - (`CDAP-425 `__). + (`CDAP-425 `__). - Improved speed of Spark unit tests - (`CDAP-600 `__). + (`CDAP-600 `__). - Spark programs now display system metrics in the CDAP Console - (`CDAP-652 `__). + (`CDAP-652 `__). - **Procedures** - Procedures have been deprecated in favor of services - (`CDAP-413 `__). + (`CDAP-413 `__). - **Services** - Added an HTTP endpoint that returns the endpoints a particular service exposes - (`CDAP-412 `__). + (`CDAP-412 `__). - Added an HTTP endpoint that lists all services - (`CDAP-469 `__). + (`CDAP-469 `__). - Default metrics for services have been added to the CDAP Console - (`CDAP-512 `__). + (`CDAP-512 `__). - The annotations ``@QueryParam`` and ``@DefaultValue`` are now supported in custom service handlers - (`CDAP-664 `__). + (`CDAP-664 `__). - **Metrics** - System and user metrics now support gauge metrics - (`CDAP-484 `__). + (`CDAP-484 `__). - Metrics can be queried using a program’s run-ID - (`CDAP-620 `__). + (`CDAP-620 `__). - **Documentation** - A `Quick Start Guide `__ has been added to the `CDAP Administration Manual `__ - (`CDAP-695 `__). + (`CDAP-695 `__). CDAP Bug Fixes -------------- - Fixed a problem with readless increments not being used when they were enabled in a dataset - (`CDAP-383 `__). + (`CDAP-383 `__). - Fixed a problem with applications, whose Spark or Scala user classes were not extended from either ``JavaSparkProgram`` or ``ScalaSparkProgram``, failing with a class loading error - (`CDAP-599 `__). + (`CDAP-599 `__). - Fixed a problem with the `CDAP upgrade tool `__ not preserving |---| for tables with readless increments enabled |---| the coprocessor - configuration during an upgrade (`CDAP-1044 `__). + configuration during an upgrade (`CDAP-1044 `__). - Fixed a problem with the readless increment implementation dropping increment cells when - a region flush or compaction occurred (`CDAP-1062 `__). + a region flush or compaction occurred (`CDAP-1062 `__). .. _known-issues-260: @@ -7838,14 +7838,14 @@ Known Issues ------------ - When running secure Hadoop clusters, metrics and debug logs from MapReduce programs are - not available (`CDAP-64 `__ and `CDAP-797 - `__). + not available (`CDAP-64 `__ and `CDAP-797 + `__). - When upgrading a cluster from an earlier version of CDAP, warning messages may appear in the master log indicating that in-transit (emitted, but not yet processed) metrics system messages could not be decoded (*Failed to decode message to MetricsRecord*). This is because of a change in the format of emitted metrics, and can result in a small amount of metrics data points being lost (`CDAP-745 - `__). + `__). - A race condition resulting in a deadlock can occur when a TwillRunnable container shutdowns while it still has ZooKeeper events to process. This occasionally surfaces when running with OpenJDK or JDK7, though not with Oracle JDK6. It is caused by a change in the @@ -7870,20 +7870,20 @@ CDAP Bug Fixes -------------- - Fixed a problem with a Coopr-provisioned secure cluster failing to start due to a classpath - issue (`CDAP-478 `__). + issue (`CDAP-478 `__). - Fixed a problem with the WISE app zip distribution not packaged correctly; a new version - (0.2.1) has been released (`CDAP-533 `__). + (0.2.1) has been released (`CDAP-533 `__). - Fixed a problem with the examples and tests incorrectly using the ByteBuffer.array - method when reading a stream event (`CDAP-549 `__). + method when reading a stream event (`CDAP-549 `__). - Fixed a problem with the Authentication Server so that it can now communicate with an LDAP - instance over SSL (`CDAP-556 `__). + instance over SSL (`CDAP-556 `__). - Fixed a problem with the program class loader to allow applications to use a different version of a library than the one that the CDAP platform uses; for example, a different - Kafka library (`CDAP-559 `__). + Kafka library (`CDAP-559 `__). - Fixed a problem with CDAP master not obtaining new delegation tokens after running for - ``hbase.auth.key.update.interval`` milliseconds (`CDAP-562 `__). + ``hbase.auth.key.update.interval`` milliseconds (`CDAP-562 `__). - Fixed a problem with the transaction not being rolled back when a user service handler throws an exception - (`CDAP-607 `__). + (`CDAP-607 `__). Other Changes ------------- @@ -7905,12 +7905,12 @@ Known Issues - Currently, applications that include Spark or Scala classes in user classes not extended from either ``JavaSparkProgram`` or ``ScalaSparkProgram`` (depending upon the language) fail with a class loading error. Spark or Scala classes should not be used outside of the - Spark program. (`CDAP-599 `__) + Spark program. (`CDAP-599 `__) - Metrics for MapReduce programs aren't populated on secure Hadoop clusters - The metric for the number of cores shown in the Resources view of the CDAP Console will be zero unless YARN has been configured to enable virtual cores - Writing to datasets through Hive is not supported in CDH4.x - (`CDAP-988 `__). + (`CDAP-988 `__). - A race condition resulting in a deadlock can occur when a TwillRunnable container shutdowns while it still has ZooKeeper events to process. This occasionally surfaces when running with OpenJDK or JDK7, though not with Oracle JDK6. It is caused by a change in the diff --git a/cdap-docs/tools/github-release-notes.py b/cdap-docs/tools/github-release-notes.py index 00959ebbf35a..410e9551362f 100755 --- a/cdap-docs/tools/github-release-notes.py +++ b/cdap-docs/tools/github-release-notes.py @@ -144,7 +144,7 @@ def build_new_line(line, issues): else: issue_links = [] for issue in issues: - issue_links.append("[%s](https://issues.cask.co/browse/%s" % (issue, issue)) + issue_links.append("[%s](https://cdap.atlassian.net/browse/%s" % (issue, issue)) return "%s (%s))\n" % (line, ','.join(issue_links)) def read_lines(input, output, version): diff --git a/cdap-docs/user-guide/source/pipelines/plugins/pom.xml b/cdap-docs/user-guide/source/pipelines/plugins/pom.xml index e1a567a2cf89..0cedff5aedfd 100644 --- a/cdap-docs/user-guide/source/pipelines/plugins/pom.xml +++ b/cdap-docs/user-guide/source/pipelines/plugins/pom.xml @@ -85,7 +85,7 @@ - https://issues.cask.co/browse/CDAP + https://cdap.atlassian.net/browse/CDAP diff --git a/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/MasterServiceMain.java b/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/MasterServiceMain.java index 21323d825935..85b077eab903 100644 --- a/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/MasterServiceMain.java +++ b/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/MasterServiceMain.java @@ -174,7 +174,7 @@ public class MasterServiceMain extends DaemonMain { static { try { - // Workaround for release of file descriptors opened by URLClassLoader - https://issues.cask.co/browse/CDAP-2841 + // Workaround for release of file descriptors opened by URLClassLoader - https://cdap.atlassian.net/browse/CDAP-2841 URLConnections.setDefaultUseCaches(false); } catch (IOException e) { LOG.error( diff --git a/cdap-master/src/main/java/org/apache/hadoop/fs/ftp/FTPInputStream.java b/cdap-master/src/main/java/org/apache/hadoop/fs/ftp/FTPInputStream.java index 3b8f1eb75135..45de56082c31 100644 --- a/cdap-master/src/main/java/org/apache/hadoop/fs/ftp/FTPInputStream.java +++ b/cdap-master/src/main/java/org/apache/hadoop/fs/ftp/FTPInputStream.java @@ -30,7 +30,7 @@ * seeks are attempted to the current position. Position equality check logic in {@link * FTPInputStream#seek} is the only change from the original class in Hadoop. This change is * required since {@link LineRecordReader} calls {@link FTPInputStream#seek} with value of 0. TODO: - * This file can be removed once https://issues.cask.co/browse/CDAP-5387 is addressed. + * This file can be removed once https://cdap.atlassian.net/browse/CDAP-5387 is addressed. */ @InterfaceAudience.Private @InterfaceStability.Unstable diff --git a/cdap-master/src/main/java/org/apache/hadoop/fs/sftp/SFTPInputStream.java b/cdap-master/src/main/java/org/apache/hadoop/fs/sftp/SFTPInputStream.java index 1356bbe7dbff..65a6b2b42608 100644 --- a/cdap-master/src/main/java/org/apache/hadoop/fs/sftp/SFTPInputStream.java +++ b/cdap-master/src/main/java/org/apache/hadoop/fs/sftp/SFTPInputStream.java @@ -31,7 +31,7 @@ * seeks are attempted to the current position. Position equality check logic in {@link * SFTPInputStream#seek} is the only change from the original class in Hadoop. This change is * required since {@link LineRecordReader} calls {@link SFTPInputStream#seek} with value of 0. TODO: - * This file can be removed once https://issues.cask.co/browse/CDAP-5387 is addressed. + * This file can be removed once https://cdap.atlassian.net/browse/CDAP-5387 is addressed. */ class SFTPInputStream extends FSInputStream { diff --git a/cdap-security/src/main/java/io/cdap/cdap/security/authorization/DelegatingPermissionManager.java b/cdap-security/src/main/java/io/cdap/cdap/security/authorization/DelegatingPermissionManager.java index 0a2be640c795..a1e600069f69 100644 --- a/cdap-security/src/main/java/io/cdap/cdap/security/authorization/DelegatingPermissionManager.java +++ b/cdap-security/src/main/java/io/cdap/cdap/security/authorization/DelegatingPermissionManager.java @@ -34,7 +34,7 @@ /** * A {@link PermissionManager} implements that delegates to the authorizer. Having this makes Guice * injection for Privilege manager simple. That reason will go away once - * https://issues.cask.co/browse/CDAP-11561 is fixed. + * https://cdap.atlassian.net/browse/CDAP-11561 is fixed. */ public class DelegatingPermissionManager implements PermissionManager { diff --git a/cdap-standalone/src/main/java/io/cdap/cdap/StandaloneMain.java b/cdap-standalone/src/main/java/io/cdap/cdap/StandaloneMain.java index 28afed21cd15..2eae701f3934 100644 --- a/cdap-standalone/src/main/java/io/cdap/cdap/StandaloneMain.java +++ b/cdap-standalone/src/main/java/io/cdap/cdap/StandaloneMain.java @@ -237,7 +237,7 @@ public Injector getInjector() { * Start the service. */ public void startUp() throws Exception { - // Workaround for release of file descriptors opened by URLClassLoader - https://issues.cask.co/browse/CDAP-2841 + // Workaround for release of file descriptors opened by URLClassLoader - https://cdap.atlassian.net/browse/CDAP-2841 URLConnections.setDefaultUseCaches(false); ConfigurationLogger.logImportantConfig(cConf); diff --git a/cdap-storage-spi/src/main/java/io/cdap/cdap/spi/data/transaction/TransactionRunners.java b/cdap-storage-spi/src/main/java/io/cdap/cdap/spi/data/transaction/TransactionRunners.java index e8d2f4cd8048..8ca959073b55 100644 --- a/cdap-storage-spi/src/main/java/io/cdap/cdap/spi/data/transaction/TransactionRunners.java +++ b/cdap-storage-spi/src/main/java/io/cdap/cdap/spi/data/transaction/TransactionRunners.java @@ -24,7 +24,7 @@ /** * Helper class for interacting with {@link TransactionRunner}. - * TODO: Figure out better way to propagate the exception: https://issues.cask.co/browse/CDAP-14736 + * TODO: Figure out better way to propagate the exception: https://cdap.atlassian.net/browse/CDAP-14736 */ @Beta public class TransactionRunners { diff --git a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/gateway/handlers/store/AppMetadataStore.java b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/gateway/handlers/store/AppMetadataStore.java index e88082ad5de0..027773c409c8 100644 --- a/cdap-watchdog/src/main/java/io/cdap/cdap/logging/gateway/handlers/store/AppMetadataStore.java +++ b/cdap-watchdog/src/main/java/io/cdap/cdap/logging/gateway/handlers/store/AppMetadataStore.java @@ -37,7 +37,7 @@ import javax.annotation.Nullable; /** - * Duplicate store class for application meatadata. JIRA https://issues.cask.co/browse/CDAP-2172 + * Duplicate store class for application meatadata. JIRA https://cdap.atlassian.net/browse/CDAP-2172 */ public class AppMetadataStore { @@ -53,7 +53,7 @@ public AppMetadataStore(StructuredTable table) { // TODO: getRun is duplicated from cdap-app-fabric AppMetadataStore class. // Any changes made here will have to be made over there too. - // JIRA https://issues.cask.co/browse/CDAP-2172 + // JIRA https://cdap.atlassian.net/browse/CDAP-2172 public RunRecordDetail getRun(ProgramReference programRef, String runId) throws IOException { // Query active run record first RunRecordDetail running = getUnfinishedRun(programRef, runId); diff --git a/cdap-watchdog/src/main/java/io/cdap/cdap/metrics/store/DefaultMetricStore.java b/cdap-watchdog/src/main/java/io/cdap/cdap/metrics/store/DefaultMetricStore.java index e12edaf9f6f8..d12b1fd9ba43 100644 --- a/cdap-watchdog/src/main/java/io/cdap/cdap/metrics/store/DefaultMetricStore.java +++ b/cdap-watchdog/src/main/java/io/cdap/cdap/metrics/store/DefaultMetricStore.java @@ -244,7 +244,7 @@ public class DefaultMetricStore implements MetricStore { this.resolutionTTLMap = builder.build(); FactTableSupplier factTableSupplier = (resolution, ignoredRollTime) -> { // roll time will be taken from configuration - // TODO: remove roll time from the supplier api, https://issues.cask.co/browse/CDAP-14730 + // TODO: remove roll time from the supplier api, https://cdap.atlassian.net/browse/CDAP-14730 return dsFactory.getOrCreateFactTable(resolution); }; this.cube = Suppliers.memoize(new Supplier() { diff --git a/pom.xml b/pom.xml index dba2cca4b257..a622337c5542 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ pom Cask Data Application Platform (CDAP) Data Application Platform for Hadoop - https://github.com/caskdata/cdap + https://github.com/cdapio/cdap CDAP