From 18fedd108421b8c3a85475885b24ecbfa1377faa Mon Sep 17 00:00:00 2001 From: Gemma Lamont Date: Tue, 3 Dec 2024 15:31:09 +0100 Subject: [PATCH] Remove dependency on common; split APOC core and APOC extended up to be less reliant on eachother --- .gitmodules | 2 +- extended-it/build.gradle | 1 - .../apoc/MissingExtraDependenciesTest.java | 7 +- .../test/java/apoc/couchbase/CouchbaseIT.java | 2 +- .../test/java/apoc/es/ElasticSearchTest.java | 25 +- .../java/apoc/es/ElasticVersionEightTest.java | 10 +- .../java/apoc/es/ElasticVersionSevenTest.java | 4 +- .../test/java/apoc/load/MySQLJdbcTest.java | 8 +- .../test/java/apoc/load/PostgresJdbcTest.java | 40 +- .../src/test/java/apoc/model/ModelTest.java | 32 +- .../test/java/apoc/mongodb/MongoDBTest.java | 8 +- .../src/test/java/apoc/mongodb/MongoTest.java | 2 +- .../test/java/apoc/mongodb/MongoTestBase.java | 6 +- .../test/java/apoc/neo4j/docker/BoltTest.java | 16 +- .../docker/CypherEnterpriseExtendedTest.java | 14 +- .../CypherProceduresClusterRoutingTest.java | 4 +- .../java/apoc/neo4j/docker/MetricsTest.java | 2 +- .../src/test/java/apoc/redis/RedisTest.java | 4 +- .../src/test/java/apoc/s3/LoadS3Test.java | 6 +- .../test/java/apoc/vectordb/ChromaDbTest.java | 2 +- .../test/java/apoc/vectordb/MilvusTest.java | 6 +- .../test/java/apoc/vectordb/QdrantTest.java | 8 +- .../test/java/apoc/vectordb/WeaviateTest.java | 35 +- .../src/test/java/apoc/xml/XmlTestUtils.java | 46 + extended/build.gradle | 4 +- .../apoc/ApocGlobalComponentsService.java | 36 + .../main/java/apoc/ExtendedApocConfig.java | 225 ++- .../ExtendedApocConfigExtensionFactory.java | 13 +- .../apoc/ExtendedApocExtensionFactory.java | 164 ++ .../apoc/ExtendedApocGlobalComponents.java | 10 +- .../ExtendedRegisterComponentFactory.java | 4 +- .../main/java/apoc/ExtendedSystemLabels.java | 3 +- .../java/apoc/ExtendedSystemPropertyKeys.java | 6 + .../java/apoc/JdbcRegistererInitFactory.java | 10 +- .../apoc/PoolExtensionFactoryExtended.java | 49 + .../src/main/java/apoc/PoolsExtended.java | 201 +++ extended/src/main/java/apoc/TTLConfig.java | 4 +- .../java/apoc/TTLConfigExtensionFactory.java | 2 +- .../java/apoc/agg/AggregationExtended.java | 6 +- extended/src/main/java/apoc/agg/Rollup.java | 4 +- .../java/apoc/algo/PathFindingExtended.java | 11 +- .../apoc/algo/PathFindingExtendedUtils.java | 95 ++ extended/src/main/java/apoc/bolt/Bolt.java | 14 +- .../main/java/apoc/bolt/BoltConnection.java | 42 +- extended/src/main/java/apoc/cache/Static.java | 22 +- .../src/main/java/apoc/coll/CollExtended.java | 4 +- .../java/apoc/coll/SetBackedListExtended.java | 134 ++ .../src/main/java/apoc/config/Config.java | 32 +- .../java/apoc/convert/ConvertExtended.java | 14 +- .../apoc/convert/ConvertExtendedUtil.java | 46 +- .../apoc/convert/ConvertUtilsExtended.java | 60 + .../main/java/apoc/couchbase/Couchbase.java | 4 +- .../java/apoc/couchbase/CouchbaseConfig.java | 40 +- .../java/apoc/couchbase/CouchbaseManager.java | 4 +- .../custom/CypherHandlerNewProcedure.java | 23 +- .../apoc/custom/CypherProceduresHandler.java | 51 +- .../apoc/custom/CypherProceduresUtil.java | 13 +- .../src/main/java/apoc/custom/Signatures.java | 6 +- .../main/java/apoc/cypher/CypherExtended.java | 94 +- .../java/apoc/cypher/CypherExtendedUtils.java | 33 + .../export/DatabaseSubGraphExtended.java | 166 ++ .../apoc/cypher/export/SubGraphExtended.java | 119 ++ .../java/apoc/data/email/ExtractEmail.java | 4 +- .../apoc/data/email/ExtractEmailHandler.java | 4 +- .../java/apoc/date/DateExtendedUtils.java | 55 + .../src/main/java/apoc/diff/DiffExtended.java | 6 +- .../apoc/dv/DataVirtualizationCatalog.java | 36 +- .../dv/DataVirtualizationCatalogHandler.java | 29 +- .../src/main/java/apoc/es/ElasticSearch.java | 52 +- .../java/apoc/es/ElasticSearchHandler.java | 6 +- .../java/apoc/export/arrow/ArrowConfig.java | 4 +- .../export/arrow/ExportArrowExtended.java | 52 +- .../export/arrow/ExportArrowFileStrategy.java | 38 +- .../apoc/export/arrow/ExportArrowService.java | 30 +- .../export/arrow/ExportArrowStrategy.java | 13 +- .../arrow/ExportArrowStreamStrategy.java | 24 +- .../export/arrow/ExportGraphFileStrategy.java | 26 +- .../export/arrow/ExportGraphStrategy.java | 14 +- .../arrow/ExportGraphStreamStrategy.java | 20 +- .../arrow/ExportResultFileStrategy.java | 10 +- .../export/arrow/ExportResultStrategy.java | 4 +- .../arrow/ExportResultStreamStrategy.java | 6 +- .../java/apoc/export/arrow/ImportArrow.java | 26 +- .../cypher/ExportFileManagerExtended.java | 32 + .../cypher/FileManagerFactoryExtended.java | 155 ++ .../AbstractCypherFormatterExtended.java | 680 ++++++++ .../AddStructureCypherFormatterExtended.java | 109 ++ .../CreateCypherFormatterExtended.java | 103 ++ .../formatter/CypherFormatExtended.java | 74 + .../formatter/CypherFormatterExtended.java | 87 ++ .../CypherFormatterUtilsExtended.java | 319 ++++ .../UpdateAllCypherFormatterExtended.java | 80 + ...pdateStructureCypherFormatterExtended.java | 106 ++ .../apoc/export/parquet/ExportParquet.java | 75 +- .../parquet/ExportParquetFileStrategy.java | 42 +- .../ExportParquetGraphFileStrategy.java | 24 +- .../ExportParquetGraphStreamStrategy.java | 15 +- .../ExportParquetResultFileStrategy.java | 12 +- .../ExportParquetResultStreamStrategy.java | 4 +- .../parquet/ExportParquetStreamStrategy.java | 32 +- .../apoc/export/parquet/ImportParquet.java | 20 +- .../apoc/export/parquet/ParquetConfig.java | 10 +- .../export/parquet/ParquetExportType.java | 20 +- .../apoc/export/parquet/ParquetReadUtil.java | 6 +- .../java/apoc/export/parquet/ParquetUtil.java | 21 +- .../apoc/export/util/BatchTransaction.java | 78 + .../export/util/BulkImportUtilExtended.java | 62 + .../util/CountingInputStreamExtended.java | 89 ++ .../export/util/CountingReaderExtended.java | 64 + .../util/DurationValueSerializerExtended.java | 38 + .../export/util/ExportConfigExtended.java | 306 ++++ .../export/util/ExportFormatExtended.java | 89 ++ .../apoc/export/util/ExportUtilsExtended.java | 72 + .../apoc/export/util/FormatUtilsExtended.java | 151 ++ .../util/LimitedSizeInputStreamExtended.java | 80 + .../export/util/MetaInformationExtended.java | 153 ++ .../util/NodesAndRelsSubGraphExtended.java | 182 +++ .../export/util/PointSerializerExtended.java | 153 ++ .../export/util/ProgressReporterExtended.java | 124 ++ .../apoc/export/util/ReporterExtended.java | 37 + .../apoc/export/util/SizeCounterExtended.java | 27 + .../util/TemporalSerializerExtended.java | 38 + .../main/java/apoc/export/xls/ExportXls.java | 34 +- .../apoc/export/xls/ExportXlsHandler.java | 32 +- .../java/apoc/export/xls/XlsExportConfig.java | 6 +- .../config/InvalidConfigException.java | 29 + extended/src/main/java/apoc/gephi/Gephi.java | 26 +- .../apoc/gephi/GephiFormatUtilsExtended.java | 61 + .../src/main/java/apoc/get/GetExtended.java | 59 + .../src/main/java/apoc/get/GetProcedures.java | 12 +- .../apoc/graph/DocumentToGraphExtended.java | 436 ++++++ .../main/java/apoc/graph/GraphsExtended.java | 18 +- .../java/apoc/graph/GraphsUtilsExtended.java | 56 + .../java/apoc/graph/LabelBuilderExtended.java | 51 + .../graph/RelationshipBuilderExtended.java | 62 + .../apoc/graph/util/GraphsConfigExtended.java | 214 +++ .../java/apoc/load/CSVResultExtended.java | 103 ++ extended/src/main/java/apoc/load/Gexf.java | 34 +- extended/src/main/java/apoc/load/Jdbc.java | 18 +- extended/src/main/java/apoc/load/LoadCsv.java | 40 +- .../main/java/apoc/load/LoadDirectory.java | 17 +- .../java/apoc/load/LoadDirectoryHandler.java | 12 +- .../src/main/java/apoc/load/LoadHtml.java | 18 +- .../main/java/apoc/load/LoadHtmlConfig.java | 46 +- .../main/java/apoc/load/LoadJsonExtended.java | 12 +- .../java/apoc/load/LoadJsonUtilsExtended.java | 74 + .../src/main/java/apoc/load/LoadLdap.java | 10 +- .../src/main/java/apoc/load/LoadParquet.java | 14 +- .../main/java/apoc/load/MappingExtended.java | 132 ++ .../apoc/load/arrow/LoadArrowExtended.java | 38 +- .../main/java/apoc/load/util/JdbcUtil.java | 6 +- .../java/apoc/load/util/LoadCsvConfig.java | 26 +- .../java/apoc/load/util/LoadJdbcConfig.java | 6 +- .../java/apoc/load/util/ResultsExtended.java | 26 + .../main/java/apoc/load/util/XmlReadUtil.java | 60 +- .../src/main/java/apoc/load/xls/LoadXls.java | 28 +- .../java/apoc/load/xls/LoadXlsHandler.java | 6 +- .../java/apoc/log/Neo4jLogStreamExtended.java | 4 +- .../src/main/java/apoc/map/MapsExtended.java | 4 +- .../main/java/apoc/meta/TypesExtended.java | 194 +++ .../PropertyTrackerExtended.java | 92 ++ .../src/main/java/apoc/metrics/Metrics.java | 28 +- .../src/main/java/apoc/ml/MixedbreadAI.java | 10 +- extended/src/main/java/apoc/ml/OpenAI.java | 24 +- .../java/apoc/ml/OpenAIRequestHandler.java | 12 +- extended/src/main/java/apoc/ml/Prompt.java | 14 +- extended/src/main/java/apoc/ml/RagConfig.java | 6 +- extended/src/main/java/apoc/ml/VertexAI.java | 50 +- .../main/java/apoc/ml/VertexAIHandler.java | 6 +- .../src/main/java/apoc/ml/aws/AWSConfig.java | 6 +- .../src/main/java/apoc/ml/aws/Bedrock.java | 30 +- .../java/apoc/ml/aws/BedrockInvokeConfig.java | 4 +- .../src/main/java/apoc/ml/aws/SageMaker.java | 28 +- .../src/main/java/apoc/ml/watson/Watson.java | 18 +- .../java/apoc/ml/watson/WatsonHandler.java | 4 +- extended/src/main/java/apoc/model/Model.java | 6 +- .../src/main/java/apoc/model/ModelConfig.java | 4 +- .../src/main/java/apoc/mongodb/Mongo.java | 28 +- .../src/main/java/apoc/mongodb/MongoDB.java | 6 +- .../main/java/apoc/mongodb/MongoDBColl.java | 6 +- .../main/java/apoc/mongodb/MongoDBUtils.java | 8 +- .../apoc/mongodb/MongoDbCollInterface.java | 4 +- .../main/java/apoc/mongodb/MongoDbConfig.java | 12 +- .../main/java/apoc/nodes/NodesExtended.java | 10 +- ...RelationshipTypeAndDirectionsExtended.java | 77 + .../BatchAndTotalCollectorExtended.java | 140 ++ .../periodic/BatchAndTotalResultExtended.java | 131 ++ .../java/apoc/periodic/BatchModeExtended.java | 63 + .../java/apoc/periodic/PeriodicExtended.java | 12 +- .../apoc/periodic/PeriodicUtilsExtended.java | 373 +++++ extended/src/main/java/apoc/redis/Redis.java | 112 +- .../src/main/java/apoc/redis/RedisConfig.java | 10 +- .../apoc/result/ByteArrayResultExtended.java | 32 + .../result/CreatedNodeResultExtended.java | 41 + .../CreatedVirtualNodeResultExtended.java | 41 + ...atedVirtualRelationshipResultExtended.java | 31 + .../CypherStatementMapResultExtended.java | 39 + .../apoc/result/DataProgressInfoExtended.java | 146 ++ .../result/DistancePathResultExtended.java | 41 + .../result/ExportProgressInfoExtended.java | 137 ++ .../java/apoc/result/GraphResultExtended.java | 38 + .../result/ImportProgressInfoExtended.java | 133 ++ .../IndexConstraintNodeInfoExtended.java | 95 ++ ...dexConstraintRelationshipInfoExtended.java | 53 + .../java/apoc/result/KernelInfoResult.java | 2 +- .../java/apoc/result/ListResultExtended.java | 33 + .../result/LoadDataMapResultExtended.java | 39 + .../java/apoc/result/LongResultExtended.java | 32 + .../java/apoc/result/MapResultExtended.java | 39 + .../apoc/result/NodeListResultExtended.java | 35 + .../java/apoc/result/NodeResultExtended.java | 45 + .../result/NodeResultWithStatsExtended.java | 34 + .../apoc/result/ObjectResultExtended.java | 30 + .../java/apoc/result/PathResultExtended.java | 35 + .../apoc/result/ProgressInfoExtended.java | 44 + .../result/RelationshipResultExtended.java | 35 + .../RelationshipResultWithStatsExtended.java | 34 + .../java/apoc/result/RowResultExtended.java | 32 + .../result/UpdatedNodeResultExtended.java | 41 + .../UpdatedRelationshipResultExtended.java | 31 + .../apoc/result/VirtualGraphExtended.java | 66 + .../java/apoc/result/VirtualNodeExtended.java | 314 ++++ .../java/apoc/result/VirtualPathExtended.java | 185 +++ .../result/VirtualPathResultExtended.java | 44 + .../result/VirtualRelationshipExtended.java | 192 +++ .../result/WeightedPathResultExtended.java | 51 + .../src/main/java/apoc/systemdb/SystemDb.java | 44 +- .../metadata/ExportDataVirtualization.java | 11 +- .../systemdb/metadata/ExportFunction.java | 9 +- .../systemdb/metadata/ExportMetadata.java | 13 +- .../systemdb/metadata/ExportProcedure.java | 9 +- .../apoc/systemdb/metadata/ExportTrigger.java | 18 +- .../apoc/systemdb/metadata/ExportUuid.java | 4 +- .../java/apoc/trigger/TriggerExtended.java | 14 +- extended/src/main/java/apoc/ttl/TTL.java | 6 +- .../src/main/java/apoc/ttl/TTLLifeCycle.java | 8 +- .../util/ApocUrlStreamHandlerFactory.java | 31 + .../java/apoc/util/ArchiveTypeExtended.java | 67 + .../util/ArrayBackedIteratorExtended.java | 48 + .../apoc/util/ArrayBackedListExtended.java | 48 + .../apoc/util/CollectionUtilsExtended.java | 72 + .../apoc/util/CompressionAlgoExtended.java | 102 ++ .../apoc/util/CompressionConfigExtended.java | 51 + .../apoc/util/DateFormatUtilExtended.java | 132 ++ .../java/apoc/util/DateParseUtilExtended.java | 93 ++ .../apoc/util/DurationFormatUtilExtended.java | 128 ++ .../java/apoc/util/EntityUtilExtended.java | 55 + .../java/apoc/util/ExtendedFileUtils.java | 11 +- .../src/main/java/apoc/util/ExtendedUtil.java | 58 +- .../java/apoc/util/FileUtilsExtended.java | 349 +++++ .../util/FixedSizeStringWriterExtended.java | 67 + .../main/java/apoc/util/JsonUtilExtended.java | 181 +++ .../main/java/apoc/util/LogsUtilExtended.java | 57 + .../main/java/apoc/util/MapUtilExtended.java | 42 + .../MissingDependencyExceptionExtended.java | 28 + .../util/QueueBasedSpliteratorExtended.java | 71 + .../java/apoc/util/QueueUtilExtended.java | 89 ++ .../apoc/util/SortedArraySetExtended.java | 83 + .../apoc/util/StreamConnectionExtended.java | 132 ++ .../apoc/util/SupportedProtocolsExtended.java | 46 + .../src/main/java/apoc/util/SystemDbUtil.java | 24 +- .../src/main/java/apoc/util/UriResolver.java | 5 +- .../src/main/java/apoc/util/UrlResolver.java | 4 +- .../src/main/java/apoc/util/UtilExtended.java | 1380 +++++++++++++++++ .../AbstractResourceIterableExtended.java | 170 ++ .../collection/FilteringIterableExtended.java | 43 + .../collection/FilteringIteratorExtended.java | 49 + .../util/collection/IterablesExtended.java | 257 +++ .../util/collection/IteratorsExtended.java | 227 +++ .../NestingResourceIteratorExtended.java | 56 + .../PrefetchingIteratorExtended.java | 84 + .../PrefetchingResourceIteratorExtended.java | 23 + .../ResourceClosingIteratorExtended.java | 97 ++ ...eIteratorCloseFailedExceptionExtended.java | 35 + .../WrappingResourceIteratorExtended.java | 44 + .../cloud/GCStorageURLConnectionExtended.java | 122 ++ .../GCStorageURLStreamHandlerExtended.java | 32 + ...torageURLStreamHandlerFactoryExtended.java | 36 + .../apoc/util/hdfs/HDFSUtilsExtended.java | 82 + ...ThreadedGlobalGraphOperationsExtended.java | 158 ++ .../main/java/apoc/util/s3/S3AwsExtended.java | 110 ++ .../apoc/util/s3/S3OutputStreamExtended.java | 310 ++++ .../java/apoc/util/s3/S3ParamsExtended.java | 75 + .../util/s3/S3ParamsExtractorExtended.java | 139 ++ .../apoc/util/s3/S3URLConnectionExtended.java | 67 + .../apoc/util/s3/S3UploadUtilsExtended.java | 37 + .../s3/S3UrlStreamHandlerFactoryExtended.java | 37 + .../apoc/uuid/UUIDHandlerNewProcedures.java | 11 +- .../java/apoc/uuid/UUIDNewProcedures.java | 4 +- extended/src/main/java/apoc/uuid/Uuid.java | 12 +- .../src/main/java/apoc/uuid/UuidConfig.java | 2 +- .../src/main/java/apoc/uuid/UuidHandler.java | 41 +- .../main/java/apoc/uuid/UuidUtilExtended.java | 79 + .../src/main/java/apoc/vectordb/ChromaDb.java | 50 +- .../java/apoc/vectordb/ChromaHandler.java | 2 +- .../src/main/java/apoc/vectordb/Milvus.java | 30 +- .../java/apoc/vectordb/MilvusHandler.java | 2 +- .../src/main/java/apoc/vectordb/Pinecone.java | 34 +- .../java/apoc/vectordb/PineconeHandler.java | 5 +- .../src/main/java/apoc/vectordb/Qdrant.java | 30 +- .../java/apoc/vectordb/QdrantHandler.java | 2 +- .../src/main/java/apoc/vectordb/VectorDb.java | 25 +- .../main/java/apoc/vectordb/VectorDbUtil.java | 11 +- .../apoc/vectordb/VectorEmbeddingConfig.java | 6 +- .../src/main/java/apoc/vectordb/Weaviate.java | 50 +- .../java/apoc/vectordb/WeaviateHandler.java | 2 +- .../src/main/kotlin/apoc/ai/service/AI.kt | 8 +- .../main/kotlin/apoc/nlp/NLPVirtualGraph.kt | 8 +- .../main/kotlin/apoc/nlp/aws/AWSProcedures.kt | 16 +- .../apoc/nlp/aws/AWSVirtualEntitiesGraph.kt | 17 +- .../apoc/nlp/aws/AWSVirtualKeyPhrasesGraph.kt | 17 +- .../aws/AWSVirtualSentimentVirtualGraph.kt | 11 +- .../main/kotlin/apoc/nlp/aws/RealAWSClient.kt | 28 +- .../kotlin/apoc/nlp/azure/AzureProcedures.kt | 8 +- .../azure/azure/AzureVirtualEntitiesGraph.kt | 17 +- .../azure/AzureVirtualKeyPhrasesGraph.kt | 17 +- .../AzureVirtualSentimentVirtualGraph.kt | 11 +- .../apoc/nlp/azure/azure/RealAzureClient.kt | 6 +- .../main/kotlin/apoc/nlp/gcp/GCPProcedures.kt | 6 +- .../nlp/gcp/GCPVirtualClassificationGraph.kt | 17 +- .../apoc/nlp/gcp/GCPVirtualEntitiesGraph.kt | 17 +- .../main/kotlin/apoc/nlp/gcp/RealGCPClient.kt | 4 +- .../java/apoc/ComparePerformancesTest.java | 10 +- .../java/apoc/ExtendedApocConfigTest.java | 23 +- .../src/test/java/apoc/TTLConfigTest.java | 6 +- .../test/java/apoc/agg/MultiStatsTest.java | 6 +- .../src/test/java/apoc/agg/RollupTest.java | 2 +- .../test/java/apoc/agg/RollupTestUtil.java | 2 +- .../test/java/apoc/algo/AlgoUtilExtended.java | 55 + .../apoc/algo/PathFindingExtendedTest.java | 4 +- .../src/test/java/apoc/cache/StaticTest.java | 2 +- .../apoc/coll/SetBackedListExtendedTest.java | 130 ++ .../src/test/java/apoc/config/ConfigTest.java | 40 +- .../apoc/convert/ConvertExtendedTest.java | 8 +- .../custom/CustomNewProcedureStorageTest.java | 14 +- .../apoc/custom/CypherNewProceduresTest.java | 12 +- .../custom/CypherProceduresStorageTest.java | 17 +- .../apoc/custom/CypherProceduresTest.java | 12 +- .../test/java/apoc/custom/SignatureTest.java | 4 +- .../java/apoc/cypher/CypherExtendedTest.java | 28 +- .../test/java/apoc/diff/DiffExtendedTest.java | 6 +- .../export/ExportExtendedSecurityTest.java | 12 +- .../apoc/export/arrow/ArrowExtendedTest.java | 20 +- .../export/arrow/ImportArrowExtendedTest.java | 10 +- .../java/apoc/export/csv/ExportCsvTest.java | 10 +- .../java/apoc/export/csv/ExportXlsTest.java | 42 +- .../java/apoc/export/parquet/ParquetTest.java | 24 +- .../apoc/export/parquet/ParquetTestUtil.java | 18 +- ...abasiAlbertGeneratorParameterizedTest.java | 6 +- .../generate/ErdosRenyiGeneratorTest.java | 6 +- .../test/java/apoc/generate/GenerateTest.java | 4 +- .../generate/Neo4jGraphGeneratorTest.java | 2 +- .../generate/WattsStrogatzGeneratorTest.java | 2 +- .../src/test/java/apoc/gephi/GephiTest.java | 2 +- .../test/java/apoc/get/GetProceduresTest.java | 2 +- .../java/apoc/graph/GraphsExtendedTest.java | 2 +- .../test/java/apoc/help/HelpExtendedTest.java | 2 +- .../test/java/apoc/load/AbstractJdbcTest.java | 4 +- .../java/apoc/load/CassandraJdbcTest.java | 26 +- .../src/test/java/apoc/load/GexfTest.java | 12 +- .../src/test/java/apoc/load/JdbcTest.java | 42 +- .../src/test/java/apoc/load/LoadCsvTest.java | 32 +- .../java/apoc/load/LoadDirectoryTest.java | 46 +- .../apoc/load/LoadExtendedSecurityTest.java | 28 +- .../apoc/load/LoadGoogleCloudStorageTest.java | 2 +- .../src/test/java/apoc/load/LoadHdfsTest.java | 8 +- .../src/test/java/apoc/load/LoadHtmlTest.java | 8 +- .../apoc/load/LoadHtmlTestParameterized.java | 8 +- .../src/test/java/apoc/load/LoadJsonTest.java | 32 +- .../src/test/java/apoc/load/LoadLdapTest.java | 12 +- .../load/relative/LoadRelativePathTest.java | 8 +- .../test/java/apoc/load/xls/LoadXlsTest.java | 50 +- .../apoc/log/Neo4jLogStreamExtendedTest.java | 4 +- .../src/test/java/apoc/ml/MixedbreadAIIT.java | 2 +- extended/src/test/java/apoc/ml/OpenAIIT.java | 4 +- .../test/java/apoc/ml/OpenAILLMaticIT.java | 10 +- .../test/java/apoc/ml/OpenAILocalAIIT.java | 10 +- .../src/test/java/apoc/ml/OpenAITest.java | 8 +- extended/src/test/java/apoc/ml/PromptIT.java | 14 +- .../src/test/java/apoc/ml/VertexAIIT.java | 4 +- .../src/test/java/apoc/ml/VertexAITest.java | 6 +- .../src/test/java/apoc/ml/WatsonTest.java | 10 +- .../src/test/java/apoc/ml/aws/BedrockIT.java | 16 +- .../test/java/apoc/ml/aws/BedrockTest.java | 10 +- .../java/apoc/ml/sagemaker/SageMakerIT.java | 18 +- .../java/apoc/ml/sagemaker/SageMakerTest.java | 6 +- .../test/java/apoc/ml/watson/WatsonIT.java | 4 +- .../test/java/apoc/systemdb/SystemDbTest.java | 35 +- extended/src/test/java/apoc/ttl/TTLTest.java | 8 +- .../test/java/apoc/util/BinaryTestUtil.java | 6 +- .../test/java/apoc/util/ExtendedTestUtil.java | 6 +- .../test/java/apoc/util/FileUtilsTest.java | 26 +- .../java/apoc/uuid/UUIDNewProceduresTest.java | 24 +- .../src/test/java/apoc/uuid/UUIDTest.java | 10 +- .../apoc/vectordb/PineconeCustomTest.java | 5 +- .../test/java/apoc/vectordb/PineconeTest.java | 22 +- .../java/apoc/vectordb/VectorDbTestUtil.java | 8 +- .../kotlin/apoc/nlp/NLPHelperFunctionsTest.kt | 39 +- .../nlp/NLPHelperFunctionsWithDatabaseTest.kt | 18 +- .../test/kotlin/apoc/nlp/NodeMatcherTest.kt | 52 +- extended/src/test/kotlin/apoc/nlp/NplUtils.kt | 5 +- .../apoc/nlp/RelationshipMatcherTest.kt | 73 +- .../AWSProceduresAPIWithDummyClientTest.kt | 41 +- .../aws/AWSVirtualEntitiesGraphStoreTest.kt | 22 +- .../nlp/aws/AWSVirtualEntitiesGraphTest.kt | 102 +- .../nlp/aws/AWSVirtualKeyPhrasesGraphTest.kt | 107 +- .../aws/AWSVirtualSentimentGraphStoreTest.kt | 5 +- ...rtualSentimentVirtualGraphExtendedTest.kt} | 7 +- .../AzureProceduresAPIWithDummyClientTest.kt | 32 +- .../azure/AzureVirtualEntitiesGraphTest.kt | 102 +- .../azure/AzureVirtualKeyPhrasesGraphTest.kt | 81 +- .../AzureVirtualSentimentGraphStoreTest.kt | 5 +- .../GCPProceduresAPIWithDummyClientTest.kt | 38 +- .../nlp/gcp/GCPVirtualCategoriesGraphTest.kt | 103 +- .../gcp/GCPVirtualEntitiesGraphStoreTest.kt | 22 +- .../nlp/gcp/GCPVirtualEntitiesGraphTest.kt | 102 +- settings.gradle | 6 +- 417 files changed, 18211 insertions(+), 2240 deletions(-) create mode 100644 extended-it/src/test/java/apoc/xml/XmlTestUtils.java create mode 100644 extended/src/main/java/apoc/ApocGlobalComponentsService.java create mode 100644 extended/src/main/java/apoc/ExtendedApocExtensionFactory.java create mode 100644 extended/src/main/java/apoc/PoolExtensionFactoryExtended.java create mode 100644 extended/src/main/java/apoc/PoolsExtended.java create mode 100644 extended/src/main/java/apoc/algo/PathFindingExtendedUtils.java create mode 100644 extended/src/main/java/apoc/coll/SetBackedListExtended.java create mode 100644 extended/src/main/java/apoc/convert/ConvertUtilsExtended.java create mode 100644 extended/src/main/java/apoc/cypher/CypherExtendedUtils.java create mode 100644 extended/src/main/java/apoc/cypher/export/DatabaseSubGraphExtended.java create mode 100644 extended/src/main/java/apoc/cypher/export/SubGraphExtended.java create mode 100644 extended/src/main/java/apoc/date/DateExtendedUtils.java create mode 100644 extended/src/main/java/apoc/export/cypher/ExportFileManagerExtended.java create mode 100644 extended/src/main/java/apoc/export/cypher/FileManagerFactoryExtended.java create mode 100644 extended/src/main/java/apoc/export/cypher/formatter/AbstractCypherFormatterExtended.java create mode 100644 extended/src/main/java/apoc/export/cypher/formatter/AddStructureCypherFormatterExtended.java create mode 100644 extended/src/main/java/apoc/export/cypher/formatter/CreateCypherFormatterExtended.java create mode 100644 extended/src/main/java/apoc/export/cypher/formatter/CypherFormatExtended.java create mode 100644 extended/src/main/java/apoc/export/cypher/formatter/CypherFormatterExtended.java create mode 100644 extended/src/main/java/apoc/export/cypher/formatter/CypherFormatterUtilsExtended.java create mode 100644 extended/src/main/java/apoc/export/cypher/formatter/UpdateAllCypherFormatterExtended.java create mode 100644 extended/src/main/java/apoc/export/cypher/formatter/UpdateStructureCypherFormatterExtended.java create mode 100644 extended/src/main/java/apoc/export/util/BatchTransaction.java create mode 100644 extended/src/main/java/apoc/export/util/BulkImportUtilExtended.java create mode 100644 extended/src/main/java/apoc/export/util/CountingInputStreamExtended.java create mode 100644 extended/src/main/java/apoc/export/util/CountingReaderExtended.java create mode 100644 extended/src/main/java/apoc/export/util/DurationValueSerializerExtended.java create mode 100644 extended/src/main/java/apoc/export/util/ExportConfigExtended.java create mode 100644 extended/src/main/java/apoc/export/util/ExportFormatExtended.java create mode 100644 extended/src/main/java/apoc/export/util/ExportUtilsExtended.java create mode 100644 extended/src/main/java/apoc/export/util/FormatUtilsExtended.java create mode 100644 extended/src/main/java/apoc/export/util/LimitedSizeInputStreamExtended.java create mode 100644 extended/src/main/java/apoc/export/util/MetaInformationExtended.java create mode 100644 extended/src/main/java/apoc/export/util/NodesAndRelsSubGraphExtended.java create mode 100644 extended/src/main/java/apoc/export/util/PointSerializerExtended.java create mode 100644 extended/src/main/java/apoc/export/util/ProgressReporterExtended.java create mode 100644 extended/src/main/java/apoc/export/util/ReporterExtended.java create mode 100644 extended/src/main/java/apoc/export/util/SizeCounterExtended.java create mode 100644 extended/src/main/java/apoc/export/util/TemporalSerializerExtended.java create mode 100644 extended/src/main/java/apoc/generate/config/InvalidConfigException.java create mode 100644 extended/src/main/java/apoc/gephi/GephiFormatUtilsExtended.java create mode 100644 extended/src/main/java/apoc/get/GetExtended.java create mode 100644 extended/src/main/java/apoc/graph/DocumentToGraphExtended.java create mode 100644 extended/src/main/java/apoc/graph/GraphsUtilsExtended.java create mode 100644 extended/src/main/java/apoc/graph/LabelBuilderExtended.java create mode 100644 extended/src/main/java/apoc/graph/RelationshipBuilderExtended.java create mode 100644 extended/src/main/java/apoc/graph/util/GraphsConfigExtended.java create mode 100644 extended/src/main/java/apoc/load/CSVResultExtended.java create mode 100644 extended/src/main/java/apoc/load/LoadJsonUtilsExtended.java create mode 100644 extended/src/main/java/apoc/load/MappingExtended.java create mode 100644 extended/src/main/java/apoc/load/util/ResultsExtended.java create mode 100644 extended/src/main/java/apoc/meta/TypesExtended.java create mode 100644 extended/src/main/java/apoc/meta/tablesforlabels/PropertyTrackerExtended.java create mode 100644 extended/src/main/java/apoc/path/RelationshipTypeAndDirectionsExtended.java create mode 100644 extended/src/main/java/apoc/periodic/BatchAndTotalCollectorExtended.java create mode 100644 extended/src/main/java/apoc/periodic/BatchAndTotalResultExtended.java create mode 100644 extended/src/main/java/apoc/periodic/BatchModeExtended.java create mode 100644 extended/src/main/java/apoc/periodic/PeriodicUtilsExtended.java create mode 100644 extended/src/main/java/apoc/result/ByteArrayResultExtended.java create mode 100644 extended/src/main/java/apoc/result/CreatedNodeResultExtended.java create mode 100644 extended/src/main/java/apoc/result/CreatedVirtualNodeResultExtended.java create mode 100644 extended/src/main/java/apoc/result/CreatedVirtualRelationshipResultExtended.java create mode 100644 extended/src/main/java/apoc/result/CypherStatementMapResultExtended.java create mode 100644 extended/src/main/java/apoc/result/DataProgressInfoExtended.java create mode 100644 extended/src/main/java/apoc/result/DistancePathResultExtended.java create mode 100644 extended/src/main/java/apoc/result/ExportProgressInfoExtended.java create mode 100644 extended/src/main/java/apoc/result/GraphResultExtended.java create mode 100644 extended/src/main/java/apoc/result/ImportProgressInfoExtended.java create mode 100644 extended/src/main/java/apoc/result/IndexConstraintNodeInfoExtended.java create mode 100644 extended/src/main/java/apoc/result/IndexConstraintRelationshipInfoExtended.java create mode 100644 extended/src/main/java/apoc/result/ListResultExtended.java create mode 100644 extended/src/main/java/apoc/result/LoadDataMapResultExtended.java create mode 100644 extended/src/main/java/apoc/result/LongResultExtended.java create mode 100644 extended/src/main/java/apoc/result/MapResultExtended.java create mode 100644 extended/src/main/java/apoc/result/NodeListResultExtended.java create mode 100644 extended/src/main/java/apoc/result/NodeResultExtended.java create mode 100644 extended/src/main/java/apoc/result/NodeResultWithStatsExtended.java create mode 100644 extended/src/main/java/apoc/result/ObjectResultExtended.java create mode 100644 extended/src/main/java/apoc/result/PathResultExtended.java create mode 100644 extended/src/main/java/apoc/result/ProgressInfoExtended.java create mode 100644 extended/src/main/java/apoc/result/RelationshipResultExtended.java create mode 100644 extended/src/main/java/apoc/result/RelationshipResultWithStatsExtended.java create mode 100644 extended/src/main/java/apoc/result/RowResultExtended.java create mode 100644 extended/src/main/java/apoc/result/UpdatedNodeResultExtended.java create mode 100644 extended/src/main/java/apoc/result/UpdatedRelationshipResultExtended.java create mode 100644 extended/src/main/java/apoc/result/VirtualGraphExtended.java create mode 100644 extended/src/main/java/apoc/result/VirtualNodeExtended.java create mode 100644 extended/src/main/java/apoc/result/VirtualPathExtended.java create mode 100644 extended/src/main/java/apoc/result/VirtualPathResultExtended.java create mode 100644 extended/src/main/java/apoc/result/VirtualRelationshipExtended.java create mode 100644 extended/src/main/java/apoc/result/WeightedPathResultExtended.java create mode 100644 extended/src/main/java/apoc/util/ApocUrlStreamHandlerFactory.java create mode 100644 extended/src/main/java/apoc/util/ArchiveTypeExtended.java create mode 100644 extended/src/main/java/apoc/util/ArrayBackedIteratorExtended.java create mode 100644 extended/src/main/java/apoc/util/ArrayBackedListExtended.java create mode 100644 extended/src/main/java/apoc/util/CollectionUtilsExtended.java create mode 100644 extended/src/main/java/apoc/util/CompressionAlgoExtended.java create mode 100644 extended/src/main/java/apoc/util/CompressionConfigExtended.java create mode 100644 extended/src/main/java/apoc/util/DateFormatUtilExtended.java create mode 100644 extended/src/main/java/apoc/util/DateParseUtilExtended.java create mode 100644 extended/src/main/java/apoc/util/DurationFormatUtilExtended.java create mode 100644 extended/src/main/java/apoc/util/EntityUtilExtended.java create mode 100644 extended/src/main/java/apoc/util/FileUtilsExtended.java create mode 100644 extended/src/main/java/apoc/util/FixedSizeStringWriterExtended.java create mode 100755 extended/src/main/java/apoc/util/JsonUtilExtended.java create mode 100644 extended/src/main/java/apoc/util/LogsUtilExtended.java create mode 100644 extended/src/main/java/apoc/util/MapUtilExtended.java create mode 100644 extended/src/main/java/apoc/util/MissingDependencyExceptionExtended.java create mode 100644 extended/src/main/java/apoc/util/QueueBasedSpliteratorExtended.java create mode 100644 extended/src/main/java/apoc/util/QueueUtilExtended.java create mode 100644 extended/src/main/java/apoc/util/SortedArraySetExtended.java create mode 100644 extended/src/main/java/apoc/util/StreamConnectionExtended.java create mode 100644 extended/src/main/java/apoc/util/SupportedProtocolsExtended.java create mode 100644 extended/src/main/java/apoc/util/UtilExtended.java create mode 100644 extended/src/main/java/apoc/util/collection/AbstractResourceIterableExtended.java create mode 100644 extended/src/main/java/apoc/util/collection/FilteringIterableExtended.java create mode 100644 extended/src/main/java/apoc/util/collection/FilteringIteratorExtended.java create mode 100644 extended/src/main/java/apoc/util/collection/IterablesExtended.java create mode 100644 extended/src/main/java/apoc/util/collection/IteratorsExtended.java create mode 100644 extended/src/main/java/apoc/util/collection/NestingResourceIteratorExtended.java create mode 100644 extended/src/main/java/apoc/util/collection/PrefetchingIteratorExtended.java create mode 100644 extended/src/main/java/apoc/util/collection/PrefetchingResourceIteratorExtended.java create mode 100644 extended/src/main/java/apoc/util/collection/ResourceClosingIteratorExtended.java create mode 100644 extended/src/main/java/apoc/util/collection/ResourceIteratorCloseFailedExceptionExtended.java create mode 100644 extended/src/main/java/apoc/util/collection/WrappingResourceIteratorExtended.java create mode 100644 extended/src/main/java/apoc/util/google/cloud/GCStorageURLConnectionExtended.java create mode 100644 extended/src/main/java/apoc/util/google/cloud/GCStorageURLStreamHandlerExtended.java create mode 100644 extended/src/main/java/apoc/util/google/cloud/GCStorageURLStreamHandlerFactoryExtended.java create mode 100644 extended/src/main/java/apoc/util/hdfs/HDFSUtilsExtended.java create mode 100644 extended/src/main/java/apoc/util/kernel/MultiThreadedGlobalGraphOperationsExtended.java create mode 100644 extended/src/main/java/apoc/util/s3/S3AwsExtended.java create mode 100644 extended/src/main/java/apoc/util/s3/S3OutputStreamExtended.java create mode 100644 extended/src/main/java/apoc/util/s3/S3ParamsExtended.java create mode 100644 extended/src/main/java/apoc/util/s3/S3ParamsExtractorExtended.java create mode 100644 extended/src/main/java/apoc/util/s3/S3URLConnectionExtended.java create mode 100644 extended/src/main/java/apoc/util/s3/S3UploadUtilsExtended.java create mode 100644 extended/src/main/java/apoc/util/s3/S3UrlStreamHandlerFactoryExtended.java create mode 100644 extended/src/main/java/apoc/uuid/UuidUtilExtended.java create mode 100644 extended/src/test/java/apoc/algo/AlgoUtilExtended.java create mode 100644 extended/src/test/java/apoc/coll/SetBackedListExtendedTest.java rename extended/src/test/kotlin/apoc/nlp/aws/{AWSVirtualSentimentVirtualGraphTest.kt => AWSVirtualSentimentVirtualGraphExtendedTest.kt} (92%) diff --git a/.gitmodules b/.gitmodules index ea53a5a847..978cea6d1a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "apoc-core"] path = apoc-core url = https://github.com/neo4j/apoc - branch = dev + branch = dev_remove_common diff --git a/extended-it/build.gradle b/extended-it/build.gradle index 9f8a74e91c..acc9bd8354 100644 --- a/extended-it/build.gradle +++ b/extended-it/build.gradle @@ -35,7 +35,6 @@ dependencies { testImplementation project(':core') testImplementation project(':test-utils') - testImplementation project(':common') testImplementation project(':extended') testImplementation project(':core').sourceSets.test.output compileOnly project(':extended').sourceSets.main.allJava diff --git a/extended-it/src/test/java/apoc/MissingExtraDependenciesTest.java b/extended-it/src/test/java/apoc/MissingExtraDependenciesTest.java index 71ee4ff0f7..2c2c72322d 100644 --- a/extended-it/src/test/java/apoc/MissingExtraDependenciesTest.java +++ b/extended-it/src/test/java/apoc/MissingExtraDependenciesTest.java @@ -1,6 +1,6 @@ package apoc; -import apoc.util.MissingDependencyException; +import apoc.util.MissingDependencyExceptionExtended; import apoc.util.Neo4jContainerExtension; import apoc.util.TestContainerUtil; import org.junit.AfterClass; @@ -15,7 +15,6 @@ import static apoc.couchbase.Couchbase.COUCHBASE_MISSING_DEPS_ERROR; import static apoc.data.email.ExtractEmail.EMAIL_MISSING_DEPS_ERROR; -import static apoc.export.parquet.ParquetConfig.PARQUET_MISSING_DEPS_ERROR; import static apoc.export.xls.ExportXlsHandler.XLS_MISSING_DEPS_ERROR; import static apoc.load.LoadHtml.SELENIUM_MISSING_DEPS_ERROR; import static apoc.mongodb.MongoDBUtils.MONGO_MISSING_DEPS_ERROR; @@ -27,7 +26,7 @@ /** * This test verifies that, if the `extra-dependencies` jars are not present, - * the procedures that require them fail with {@link apoc.util.MissingDependencyException} + * the procedures that require them fail with {@link MissingDependencyExceptionExtended} */ public class MissingExtraDependenciesTest { private static Neo4jContainerExtension neo4jContainer; @@ -155,7 +154,7 @@ public static void assertFails(String query, Map params, String } catch (RuntimeException e) { String message = e.getMessage(); // String of type `apoc.util.MissingDependencyException: ` - String expected = "%s: %s".formatted(MissingDependencyException.class.getName(), errorMessage); + String expected = "%s: %s".formatted(MissingDependencyExceptionExtended.class.getName(), errorMessage); assertTrue("Actual error message is: " + message, message.contains(expected)); } } diff --git a/extended-it/src/test/java/apoc/couchbase/CouchbaseIT.java b/extended-it/src/test/java/apoc/couchbase/CouchbaseIT.java index 65502848e6..f5400915a9 100644 --- a/extended-it/src/test/java/apoc/couchbase/CouchbaseIT.java +++ b/extended-it/src/test/java/apoc/couchbase/CouchbaseIT.java @@ -42,7 +42,7 @@ import static apoc.couchbase.CouchbaseTestUtils.getNumConnections; import static apoc.util.TestUtil.testCall; import static apoc.util.TestUtil.testCallEmpty; -import static apoc.util.Util.map; +import static apoc.util.UtilExtended.map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; diff --git a/extended-it/src/test/java/apoc/es/ElasticSearchTest.java b/extended-it/src/test/java/apoc/es/ElasticSearchTest.java index dbe8748a89..5be71cee39 100644 --- a/extended-it/src/test/java/apoc/es/ElasticSearchTest.java +++ b/extended-it/src/test/java/apoc/es/ElasticSearchTest.java @@ -1,9 +1,8 @@ package apoc.es; -import apoc.es.ElasticSearch; -import apoc.util.JsonUtil; +import apoc.util.JsonUtilExtended; import apoc.util.TestUtil; -import apoc.util.Util; +import apoc.util.UtilExtended; import com.fasterxml.jackson.core.JsonProcessingException; import com.jayway.jsonpath.Configuration; import com.jayway.jsonpath.JsonPath; @@ -21,7 +20,7 @@ import java.util.function.Consumer; import static apoc.ApocConfig.apocConfig; -import static apoc.util.MapUtil.map; +import static apoc.util.MapUtilExtended.map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -96,7 +95,7 @@ public static void tearDown() { */ static Map createDefaultProcedureParametersWithPayloadAndId(String payload, String id) { try { - Map mapPayload = JsonUtil.OBJECT_MAPPER.readValue(payload, Map.class); + Map mapPayload = JsonUtilExtended.OBJECT_MAPPER.readValue(payload, Map.class); return addPayloadAndIdToParams(paramsWithBasicAuth, mapPayload, id); } catch (IOException e) { throw new RuntimeException(e); @@ -104,7 +103,7 @@ static Map createDefaultProcedureParametersWithPayloadAndId(Stri } static Map addPayloadAndIdToParams(Map params, Object payload, String id) { - return Util.merge(params, Util.map("payload", payload, "id", id)); + return UtilExtended.merge(params, UtilExtended.map("payload", payload, "id", id)); } private static void insertDocuments() throws JsonProcessingException { @@ -289,9 +288,9 @@ public void testFullSearchWithOtherParametersAsAString() throws Exception { */ @Test public void testPutUpdateDocument() throws IOException{ - Map doc = JsonUtil.OBJECT_MAPPER.readValue(DOCUMENT, Map.class); + Map doc = JsonUtilExtended.OBJECT_MAPPER.readValue(DOCUMENT, Map.class); doc.put("tags", Arrays.asList("awesome")); - Map params = createDefaultProcedureParametersWithPayloadAndId(JsonUtil.OBJECT_MAPPER.writeValueAsString(doc), ES_ID); + Map params = createDefaultProcedureParametersWithPayloadAndId(JsonUtilExtended.OBJECT_MAPPER.writeValueAsString(doc), ES_ID); TestUtil.testCall(db, "CALL apoc.es.put($host,$index,$type,$id,'refresh=true',$payload, $config) yield value", params, r -> { Object updated = extractValueFromResponse(r, "$.result"); assertEquals("updated", updated); @@ -307,7 +306,7 @@ public void testPutUpdateDocument() throws IOException{ public void testPutUpdateDocumentWithAuthHeader() throws IOException{ String tags = UUID.randomUUID().toString(); - Map doc = JsonUtil.OBJECT_MAPPER.readValue(DOCUMENT, Map.class); + Map doc = JsonUtilExtended.OBJECT_MAPPER.readValue(DOCUMENT, Map.class); doc.put("tags", Arrays.asList(tags)); Map params = addPayloadAndIdToParams(paramsWithBasicAuth, doc, ES_ID); TestUtil.testCall(db, "CALL apoc.es.put($host,$index,$type,$id,'refresh=true',$payload, $config) yield value", @@ -330,8 +329,8 @@ public void testPostRawCreateDocument() throws IOException { String index = UUID.randomUUID().toString(); String type = getEsType(); String id = UUID.randomUUID().toString(); - Map payload = JsonUtil.OBJECT_MAPPER.readValue("{\"ajeje\":\"Brazorf\"}", Map.class); - Map params = Util.map("host", HTTP_HOST_ADDRESS, + Map payload = JsonUtilExtended.OBJECT_MAPPER.readValue("{\"ajeje\":\"Brazorf\"}", Map.class); + Map params = UtilExtended.map("host", HTTP_HOST_ADDRESS, "index", index, "suffix", index, "type", type, @@ -362,8 +361,8 @@ public void testPostRawCreateDocument() throws IOException { public void testPostCreateDocumentWithAuthHeader() throws IOException { String index = UUID.randomUUID().toString(); String type = getEsType(); - Map payload = JsonUtil.OBJECT_MAPPER.readValue("{\"ajeje\":\"Brazorf\"}", Map.class); - Map params = Util.map("host", elastic.getHttpHostAddress(), + Map payload = JsonUtilExtended.OBJECT_MAPPER.readValue("{\"ajeje\":\"Brazorf\"}", Map.class); + Map params = UtilExtended.map("host", elastic.getHttpHostAddress(), "index", index, "type", type, "payload", payload, diff --git a/extended-it/src/test/java/apoc/es/ElasticVersionEightTest.java b/extended-it/src/test/java/apoc/es/ElasticVersionEightTest.java index 46a60fc181..458f1333d4 100644 --- a/extended-it/src/test/java/apoc/es/ElasticVersionEightTest.java +++ b/extended-it/src/test/java/apoc/es/ElasticVersionEightTest.java @@ -1,8 +1,8 @@ package apoc.es; -import apoc.util.JsonUtil; +import apoc.util.JsonUtilExtended; import apoc.util.TestUtil; -import apoc.util.Util; +import apoc.util.UtilExtended; import com.fasterxml.jackson.core.JsonProcessingException; import com.nimbusds.jose.util.Pair; import org.junit.BeforeClass; @@ -22,7 +22,7 @@ public class ElasticVersionEightTest extends ElasticSearchTest { @BeforeClass public static void setUp() throws Exception { Map config = Map.of("headers", basicAuthHeader, VERSION_KEY, ElasticSearchHandler.Version.EIGHT.name()); - Map params = Util.map("index", ES_INDEX, + Map params = UtilExtended.map("index", ES_INDEX, "id", ES_ID, "type", ES_TYPE, "config", config); String tag = "8.14.3"; @@ -180,7 +180,7 @@ private void assertPutForRRF() { payloads.forEach(payload -> { try { - Map mapPayload = JsonUtil.OBJECT_MAPPER.readValue(payload.getRight(), Map.class); + Map mapPayload = JsonUtilExtended.OBJECT_MAPPER.readValue(payload.getRight(), Map.class); paramsWithBasicAuth.put("payload", mapPayload); paramsWithBasicAuth.put("index", payload.getLeft()); TestUtil.testCall(db, "CALL apoc.es.put($host, $index, null, null, null, $payload, $config)", @@ -197,7 +197,7 @@ private void assertPutForRRF() { } private void setPayload(String payload, Map params) throws JsonProcessingException { - Map mapPayload = JsonUtil.OBJECT_MAPPER.readValue(payload, Map.class); + Map mapPayload = JsonUtilExtended.OBJECT_MAPPER.readValue(payload, Map.class); params.put("payload", mapPayload); } diff --git a/extended-it/src/test/java/apoc/es/ElasticVersionSevenTest.java b/extended-it/src/test/java/apoc/es/ElasticVersionSevenTest.java index 61700b5f26..f7316edf7f 100644 --- a/extended-it/src/test/java/apoc/es/ElasticVersionSevenTest.java +++ b/extended-it/src/test/java/apoc/es/ElasticVersionSevenTest.java @@ -1,7 +1,7 @@ package apoc.es; import apoc.util.TestUtil; -import apoc.util.Util; +import apoc.util.UtilExtended; import org.junit.BeforeClass; import org.junit.Test; @@ -21,7 +21,7 @@ public class ElasticVersionSevenTest extends ElasticSearchTest { private final static String HOST = "localhost"; public static final ElasticSearchHandler DEFAULT_HANDLER = ElasticSearchHandler.Version.DEFAULT.get(); - private static final Map defaultParams = Util.map("index", ES_INDEX, "type", ES_TYPE, "id", ES_ID); + private static final Map defaultParams = UtilExtended.map("index", ES_INDEX, "type", ES_TYPE, "id", ES_ID); @BeforeClass public static void setUp() throws Exception { diff --git a/extended-it/src/test/java/apoc/load/MySQLJdbcTest.java b/extended-it/src/test/java/apoc/load/MySQLJdbcTest.java index 8ac4dffa2e..ff3d1203d4 100644 --- a/extended-it/src/test/java/apoc/load/MySQLJdbcTest.java +++ b/extended-it/src/test/java/apoc/load/MySQLJdbcTest.java @@ -2,7 +2,7 @@ import apoc.util.s3.MySQLContainerExtension; import apoc.util.TestUtil; -import apoc.util.Util; +import apoc.util.UtilExtended; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; @@ -91,11 +91,11 @@ private static void testLoadJdbc(DbmsRule db, MySQLContainerExtension mysql) { // with the config {timezone: 'UTC'} and `preserveInstants=true&connectionTimeZone=SERVER` to make the result deterministic, // since `TIMESTAMP` values are automatically converted from the session time zone to UTC for storage, and vice versa. testCall(db, "CALL apoc.load.jdbc($url, $table, [], {timezone: 'UTC'})", - Util.map( + UtilExtended.map( "url", mysql.getJdbcUrl() + "&preserveInstants=true&connectionTimeZone=SERVER", "table", "country"), row -> { - Map expected = Util.map( + Map expected = UtilExtended.map( "Code", "NLD", "Name", "Netherlands", "Continent", "Europe", @@ -125,7 +125,7 @@ private static void testLoadJdbc(DbmsRule db, MySQLContainerExtension mysql) { private static void testIssue3496(DbmsRule db, MySQLContainerExtension mysql) { testCall(db, "CALL apoc.load.jdbc($url,'SELECT DATE(NOW()), NOW(), CURDATE(), CURTIME(), UTC_DATE(), UTC_TIME(), UTC_TIMESTAMP(), DATE(UTC_TIMESTAMP());')", - Util.map("url", mysql.getJdbcUrl()), + UtilExtended.map("url", mysql.getJdbcUrl()), r -> { Map row = (Map) r.get("row"); assertEquals(8, row.size()); diff --git a/extended-it/src/test/java/apoc/load/PostgresJdbcTest.java b/extended-it/src/test/java/apoc/load/PostgresJdbcTest.java index 5b3410c29e..f3cf69583e 100644 --- a/extended-it/src/test/java/apoc/load/PostgresJdbcTest.java +++ b/extended-it/src/test/java/apoc/load/PostgresJdbcTest.java @@ -3,7 +3,7 @@ import apoc.periodic.Periodic; import apoc.text.Strings; import apoc.util.TestUtil; -import apoc.util.Util; +import apoc.util.UtilExtended; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; @@ -53,25 +53,25 @@ public static void tearDown() throws SQLException { @Test public void testLoadJdbc() throws Exception { - testCall(db, "CALL apoc.load.jdbc($url,'PERSON',[], $config)", Util.map("url", postgress.getJdbcUrl(), - "config", Util.map("schema", "test", - "credentials", Util.map("user", postgress.getUsername(), "password", postgress.getPassword()))), + testCall(db, "CALL apoc.load.jdbc($url,'PERSON',[], $config)", UtilExtended.map("url", postgress.getJdbcUrl(), + "config", UtilExtended.map("schema", "test", + "credentials", UtilExtended.map("user", postgress.getUsername(), "password", postgress.getPassword()))), (row) -> assertResult(row)); } @Test public void testLoadJdbSelect() throws Exception { - testCall(db, "CALL apoc.load.jdbc($url,'SELECT * FROM PERSON',[], $config)", Util.map("url", postgress.getJdbcUrl(), - "config", Util.map("schema", "test", - "credentials", Util.map("user", postgress.getUsername(), "password", postgress.getPassword()))), + testCall(db, "CALL apoc.load.jdbc($url,'SELECT * FROM PERSON',[], $config)", UtilExtended.map("url", postgress.getJdbcUrl(), + "config", UtilExtended.map("schema", "test", + "credentials", UtilExtended.map("user", postgress.getUsername(), "password", postgress.getPassword()))), (row) -> assertResult(row)); } @Test public void testLoadJdbSelectWithArrays() throws Exception { - testCall(db, "CALL apoc.load.jdbc($url,'SELECT * FROM ARRAY_TABLE',[], $config)", Util.map("url", postgress.getJdbcUrl(), - "config", Util.map("schema", "test", - "credentials", Util.map("user", postgress.getUsername(), "password", postgress.getPassword()))), + testCall(db, "CALL apoc.load.jdbc($url,'SELECT * FROM ARRAY_TABLE',[], $config)", UtilExtended.map("url", postgress.getJdbcUrl(), + "config", UtilExtended.map("schema", "test", + "credentials", UtilExtended.map("user", postgress.getUsername(), "password", postgress.getPassword()))), (result) -> { Map row = (Map)result.get("row"); assertEquals("John", row.get("NAME")); @@ -85,27 +85,27 @@ public void testLoadJdbSelectWithArrays() throws Exception { @Test public void testLoadJdbcUpdate() throws Exception { testCall(db, "CALL apoc.load.jdbcUpdate($url,'UPDATE PERSON SET \"SURNAME\" = ? WHERE \"NAME\" = ?', ['DOE', 'John'], $config)", - Util.map("url", postgress.getJdbcUrl(), - "config", Util.map("schema", "test", - "credentials", Util.map("user", postgress.getUsername(), "password", postgress.getPassword()))), - (row) -> assertEquals( Util.map("count", 1 ), row.get("row"))); + UtilExtended.map("url", postgress.getJdbcUrl(), + "config", UtilExtended.map("schema", "test", + "credentials", UtilExtended.map("user", postgress.getUsername(), "password", postgress.getPassword()))), + (row) -> assertEquals( UtilExtended.map("count", 1 ), row.get("row"))); } @Test public void testLoadJdbcParams() throws Exception { testCall(db, "CALL apoc.load.jdbc($url,'SELECT * FROM PERSON WHERE \"NAME\" = ?',['John'], $config)", // YIELD row RETURN row - Util.map("url", postgress.getJdbcUrl(), - "config", Util.map("schema", "test", - "credentials", Util.map("user", postgress.getUsername(), "password", postgress.getPassword()))), + UtilExtended.map("url", postgress.getJdbcUrl(), + "config", UtilExtended.map("schema", "test", + "credentials", UtilExtended.map("user", postgress.getUsername(), "password", postgress.getPassword()))), (row) -> assertResult(row)); } @Test @Ignore("flaky") public void testIssue4141PeriodicIterateWithJdbc() throws Exception { - var config = Util.map("url", postgress.getJdbcUrl(), - "config", Util.map("schema", "test", - "credentials", Util.map("user", postgress.getUsername(), "password", postgress.getPassword()))); + var config = UtilExtended.map("url", postgress.getJdbcUrl(), + "config", UtilExtended.map("schema", "test", + "credentials", UtilExtended.map("user", postgress.getUsername(), "password", postgress.getPassword()))); String query = "WITH range(0, 100) as list UNWIND list as l CREATE (n:MyNode{id: l})"; diff --git a/extended-it/src/test/java/apoc/model/ModelTest.java b/extended-it/src/test/java/apoc/model/ModelTest.java index 09a0697a35..a5447d01b9 100644 --- a/extended-it/src/test/java/apoc/model/ModelTest.java +++ b/extended-it/src/test/java/apoc/model/ModelTest.java @@ -1,8 +1,8 @@ package apoc.model; import apoc.util.TestUtil; -import apoc.util.Util; -import apoc.util.collection.Iterators; +import apoc.util.UtilExtended; +import apoc.util.collection.IteratorsExtended; import org.junit.*; import org.junit.rules.TestName; import org.neo4j.graphdb.*; @@ -54,12 +54,12 @@ public void initDb() { @Test public void testLoadJdbcSchema() { testCall(db, "CALL apoc.model.jdbc($url, $config)", - Util.map("url", mysqlUrl, - "config", Util.map("schema", "test", - "credentials", Util.map("user", mysql.getUsername(), "password", mysql.getPassword()))), + UtilExtended.map("url", mysqlUrl, + "config", UtilExtended.map("schema", "test", + "credentials", UtilExtended.map("user", mysql.getUsername(), "password", mysql.getPassword()))), (row) -> { Long count = db.executeTransactionally("MATCH (n) RETURN count(n) AS count", Collections.emptyMap(), - result -> Iterators.single(result.columnAs("count"))); + result -> IteratorsExtended.single(result.columnAs("count"))); assertEquals(0L, count.longValue()); List nodes = (List) row.get("nodes"); List rels = (List) row.get("relationships"); @@ -107,16 +107,16 @@ public void testLoadJdbcSchema() { @Test public void testLoadJdbcSchemaWithWriteOperation() { db.executeTransactionally("CALL apoc.model.jdbc($url, $config)", - Util.map("url", mysqlUrl, - "config", Util.map("schema", "test", + UtilExtended.map("url", mysqlUrl, + "config", UtilExtended.map("schema", "test", "write", true, - "credentials", Util.map("user", mysql.getUsername(), "password", mysql.getPassword()))), - innerResult -> Iterators.single(innerResult) + "credentials", UtilExtended.map("user", mysql.getUsername(), "password", mysql.getPassword()))), + innerResult -> IteratorsExtended.single(innerResult) ); try (Transaction tx = db.beginTx()) { - List nodes = Iterators.single(tx.execute("MATCH (n) RETURN collect(distinct n) AS nodes").columnAs("nodes")); - List rels = Iterators.single(tx.execute("MATCH ()-[r]-() RETURN collect(distinct r) AS rels").columnAs("rels")); + List nodes = IteratorsExtended.single(tx.execute("MATCH (n) RETURN collect(distinct n) AS nodes").columnAs("nodes")); + List rels = IteratorsExtended.single(tx.execute("MATCH ()-[r]-() RETURN collect(distinct r) AS rels").columnAs("rels")); assertEquals( 33, nodes.size()); assertEquals( 32, rels.size()); @@ -162,10 +162,10 @@ public void testLoadJdbcSchemaWithWriteOperation() { @Test public void testLoadJdbcSchemaWithFiltering() { testCall(db, "CALL apoc.model.jdbc($url, $config)", - Util.map("url", mysqlUrl, - "config", Util.map("schema", "test", - "credentials", Util.map("user", mysql.getUsername(), "password", mysql.getPassword()), - "filters", Util.map("tables", Arrays.asList("country\\w*"), "columns", Arrays.asList("(?i)code", "(?i)name", "(?i)Language")))), + UtilExtended.map("url", mysqlUrl, + "config", UtilExtended.map("schema", "test", + "credentials", UtilExtended.map("user", mysql.getUsername(), "password", mysql.getPassword()), + "filters", UtilExtended.map("tables", Arrays.asList("country\\w*"), "columns", Arrays.asList("(?i)code", "(?i)name", "(?i)Language")))), (row) -> { List nodes = (List) row.get("nodes"); List rels = (List) row.get("relationships"); diff --git a/extended-it/src/test/java/apoc/mongodb/MongoDBTest.java b/extended-it/src/test/java/apoc/mongodb/MongoDBTest.java index cb09af597d..f0c4a22f0d 100644 --- a/extended-it/src/test/java/apoc/mongodb/MongoDBTest.java +++ b/extended-it/src/test/java/apoc/mongodb/MongoDBTest.java @@ -1,7 +1,7 @@ package apoc.mongodb; import apoc.graph.Graphs; -import apoc.util.MapUtil; +import apoc.util.MapUtilExtended; import apoc.util.TestUtil; import apoc.util.UrlResolver; import com.mongodb.client.MongoClient; @@ -23,7 +23,7 @@ import java.util.Set; import java.util.stream.Collectors; -import static apoc.util.MapUtil.map; +import static apoc.util.MapUtilExtended.map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -91,7 +91,7 @@ public void testObjectIdToStringMapping() { String url = new UrlResolver("mongodb", mongo.getContainerIpAddress(), mongo.getMappedPort(MONGO_DEFAULT_PORT)) .getUrl("mongodb", mongo.getContainerIpAddress()); try (MongoDbCollInterface coll = MongoDbCollInterface.Factory.create(url, "test", "person", false, false, false)) { - Map document = coll.first(MapUtil.map("name", "Andrea Santurbano")); + Map document = coll.first(MapUtilExtended.map("name", "Andrea Santurbano")); assertTrue(document.get("_id") instanceof String); Collection bought = (Collection) document.get("bought"); assertEquals(2, bought.size()); @@ -107,7 +107,7 @@ public void testCompatibleValues() { String url = new UrlResolver("mongodb", mongo.getContainerIpAddress(), mongo.getMappedPort(MONGO_DEFAULT_PORT)) .getUrl("mongodb", mongo.getContainerIpAddress()); try (MongoDbCollInterface coll = MongoDbCollInterface.Factory.create(url, "test", "test", true, false, true)) { - Map document = coll.first(MapUtil.map("name", "testDocument")); + Map document = coll.first(MapUtilExtended.map("name", "testDocument")); assertNotNull(((Map) document.get("_id")).get("timestamp")); assertEquals(LocalDateTime.from(currentTime.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime()), document.get("date")); assertEquals(longValue, document.get("longValue")); diff --git a/extended-it/src/test/java/apoc/mongodb/MongoTest.java b/extended-it/src/test/java/apoc/mongodb/MongoTest.java index c5c6271dfd..9571a37560 100644 --- a/extended-it/src/test/java/apoc/mongodb/MongoTest.java +++ b/extended-it/src/test/java/apoc/mongodb/MongoTest.java @@ -36,7 +36,7 @@ import java.util.function.Consumer; import static apoc.mongodb.MongoDBColl.ERROR_MESSAGE; -import static apoc.util.MapUtil.map; +import static apoc.util.MapUtilExtended.map; import static apoc.util.TestUtil.testCall; import static apoc.util.TestUtil.testResult; import static org.junit.Assert.assertEquals; diff --git a/extended-it/src/test/java/apoc/mongodb/MongoTestBase.java b/extended-it/src/test/java/apoc/mongodb/MongoTestBase.java index 1c1a84ed7b..8d29d94d98 100644 --- a/extended-it/src/test/java/apoc/mongodb/MongoTestBase.java +++ b/extended-it/src/test/java/apoc/mongodb/MongoTestBase.java @@ -1,6 +1,6 @@ package apoc.mongodb; -import apoc.util.JsonUtil; +import apoc.util.JsonUtilExtended; import com.mongodb.client.MongoClient; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; @@ -41,7 +41,7 @@ import java.util.stream.Stream; import java.util.stream.StreamSupport; -import static apoc.util.MapUtil.map; +import static apoc.util.MapUtilExtended.map; import static java.net.HttpURLConnection.HTTP_OK; import static java.net.HttpURLConnection.HTTP_UNAUTHORIZED; import static org.junit.Assert.assertArrayEquals; @@ -185,7 +185,7 @@ public static Map getNumConnections(GenericContainer mongo, Stri .collect(Collectors.toList()); lists = lists.subList(lists.indexOf("{"), lists.size()); String jsonStr = String.join("", lists); - return JsonUtil.OBJECT_MAPPER.readValue(jsonStr, Map.class); + return JsonUtilExtended.OBJECT_MAPPER.readValue(jsonStr, Map.class); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/extended-it/src/test/java/apoc/neo4j/docker/BoltTest.java b/extended-it/src/test/java/apoc/neo4j/docker/BoltTest.java index 508f405d5b..d38fd3f704 100644 --- a/extended-it/src/test/java/apoc/neo4j/docker/BoltTest.java +++ b/extended-it/src/test/java/apoc/neo4j/docker/BoltTest.java @@ -9,7 +9,7 @@ import apoc.util.TestContainerUtil; import apoc.util.TestContainerUtil.ApocPackage; import apoc.util.TestUtil; -import apoc.util.Util; +import apoc.util.UtilExtended; import org.junit.After; import org.junit.AfterClass; import org.junit.Assume; @@ -532,9 +532,9 @@ public void testExecuteCreateNodeStatistic() throws Exception { assertNotNull(r); Map row = r.next(); Map result = (Map) row.get("row"); - assertEquals(1L, (long) Util.toLong(result.get("nodesCreated"))); - assertEquals(1L, (long) Util.toLong(result.get("labelsAdded"))); - assertEquals(1L, (long) Util.toLong(result.get("propertiesSet"))); + assertEquals(1L, (long) UtilExtended.toLong(result.get("nodesCreated"))); + assertEquals(1L, (long) UtilExtended.toLong(result.get("labelsAdded"))); + assertEquals(1L, (long) UtilExtended.toLong(result.get("propertiesSet"))); assertEquals(false, r.hasNext()); }); } @@ -640,11 +640,11 @@ public void testLoadBigPathVirtual() throws Exception { public void testLoadFromLocal() { String localStatement = "RETURN 'foobar' AS foobar"; String remoteStatement = "CREATE (n: TestLoadFromLocalNode { m: foobar })"; - final Map map = Util.map( + final Map map = UtilExtended.map( "url", BOLT_URL, "localStatement", localStatement, "remoteStatement", remoteStatement, - "config", Util.map("readOnly", false)); + "config", UtilExtended.map("readOnly", false)); db.executeTransactionally("call apoc.bolt.load.fromLocal($url, $localStatement, $remoteStatement, $config) YIELD row return row", map); final long remoteCount = neo4jContainer.getSession().executeRead(tx -> (long) tx.run("MATCH (n: TestLoadFromLocalNode { m: 'foobar' }) RETURN count(n) AS count").single().asMap().get("count")); @@ -654,11 +654,11 @@ public void testLoadFromLocal() { @Test public void testLoadFromLocalStream() { String localStatement = "RETURN \"CREATE (n: TestLoadFromLocalStream)\" AS statement"; - final Map map = Util.map( + final Map map = UtilExtended.map( "url", BOLT_URL, "localStatement", localStatement, "remoteStatement", null, - "config", Util.map("readOnly", false, "streamStatements", true)); + "config", UtilExtended.map("readOnly", false, "streamStatements", true)); db.executeTransactionally("call apoc.bolt.load.fromLocal($url, $localStatement, $remoteStatement, $config)", map); final long remoteCount = neo4jContainer.getSession().executeRead(tx -> (long) tx.run("MATCH (n: TestLoadFromLocalStream) RETURN count(n) AS count").single().asMap().get("count")); diff --git a/extended-it/src/test/java/apoc/neo4j/docker/CypherEnterpriseExtendedTest.java b/extended-it/src/test/java/apoc/neo4j/docker/CypherEnterpriseExtendedTest.java index 864eca7c6a..804e302208 100644 --- a/extended-it/src/test/java/apoc/neo4j/docker/CypherEnterpriseExtendedTest.java +++ b/extended-it/src/test/java/apoc/neo4j/docker/CypherEnterpriseExtendedTest.java @@ -2,8 +2,8 @@ import apoc.util.Neo4jContainerExtension; import apoc.util.TestContainerUtil.ApocPackage; -import apoc.util.collection.Iterables; -import apoc.util.collection.Iterators; +import apoc.util.collection.IterablesExtended; +import apoc.util.collection.IteratorsExtended; import org.apache.commons.io.FileUtils; import org.junit.After; import org.junit.AfterClass; @@ -25,7 +25,7 @@ import static apoc.util.TestContainerUtil.testCall; import static apoc.util.TestContainerUtil.testCallEmpty; import static apoc.util.TestContainerUtil.testResult; -import static apoc.util.Util.map; +import static apoc.util.UtilExtended.map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThrows; @@ -407,7 +407,7 @@ public void testRunProcedureWithSetAndReturnResults(String query, Map(:Other {updated: 'true'}) RETURN *", - r -> assertEquals(4L, Iterators.count(r)) + r -> assertEquals(4L, IteratorsExtended.count(r)) ); } @@ -438,7 +438,7 @@ private void testCypherMapParallelCommon(String query, Map param public void assertOtherNodeAndRel(long id, Map result) { Node n = (Node) result.get("o"); - assertEquals(List.of("Other"), Iterables.asList(n.labels())); + assertEquals(List.of("Other"), IterablesExtended.asList(n.labels())); assertEquals(Map.of("idOther", id), n.asMap()); Relationship rel = (Relationship) result.get("r"); @@ -471,7 +471,7 @@ private void assertReturnQueryNode(Map row, long id) { public void assertReturnQueryNode(long id, Map result) { Node n = result.get("n"); - assertEquals(List.of("ReturnQuery"), Iterables.asList(n.labels())); + assertEquals(List.of("ReturnQuery"), IterablesExtended.asList(n.labels())); assertEquals(Map.of("id", id), n.asMap()); } @@ -482,7 +482,7 @@ private void assertRunProcNode(Map row, long id) { assertEquals(1, result.size()); Node n = result.get("n"); - assertEquals(List.of("Result"), Iterables.asList(n.labels())); + assertEquals(List.of("Result"), IterablesExtended.asList(n.labels())); assertEquals(Map.of("id", id, "updated", true), n.asMap()); assertEquals(SET_RETURN_FILE, row.get("fileName")); diff --git a/extended-it/src/test/java/apoc/neo4j/docker/CypherProceduresClusterRoutingTest.java b/extended-it/src/test/java/apoc/neo4j/docker/CypherProceduresClusterRoutingTest.java index 90f48ae0e7..6b47658db9 100644 --- a/extended-it/src/test/java/apoc/neo4j/docker/CypherProceduresClusterRoutingTest.java +++ b/extended-it/src/test/java/apoc/neo4j/docker/CypherProceduresClusterRoutingTest.java @@ -3,7 +3,7 @@ import apoc.util.Neo4jContainerExtension; import apoc.util.TestContainerUtil; import apoc.util.TestcontainersCausalCluster; -import apoc.util.collection.Iterators; +import apoc.util.collection.IteratorsExtended; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Ignore; @@ -162,7 +162,7 @@ public void testCustomInstallAndDropAllowedOnlyInSysLeaderMember() { @Test public void testCustomShowAllowedInAllSysLeaderMembers() { final String query = "CALL apoc.custom.show"; - final BiConsumer testUuidShow = (session, name) -> testResult(session, query, Iterators::count); + final BiConsumer testUuidShow = (session, name) -> testResult(session, query, IteratorsExtended::count); customInSysLeaderMemberCommon(testUuidShow, true); } diff --git a/extended-it/src/test/java/apoc/neo4j/docker/MetricsTest.java b/extended-it/src/test/java/apoc/neo4j/docker/MetricsTest.java index 64ae293365..f63f1b8818 100644 --- a/extended-it/src/test/java/apoc/neo4j/docker/MetricsTest.java +++ b/extended-it/src/test/java/apoc/neo4j/docker/MetricsTest.java @@ -20,7 +20,7 @@ import static apoc.metrics.Metrics.OUTSIDE_DIR_ERR_MSG; import static apoc.util.ExtendedFileUtils.NEO4J_DIRECTORY_CONFIGURATION_SETTING_NAMES; import static apoc.util.TestContainerUtil.*; -import static apoc.util.Util.map; +import static apoc.util.UtilExtended.map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.neo4j.test.assertion.Assert.assertEventually; diff --git a/extended-it/src/test/java/apoc/redis/RedisTest.java b/extended-it/src/test/java/apoc/redis/RedisTest.java index 0d316565a8..ccb44917c4 100644 --- a/extended-it/src/test/java/apoc/redis/RedisTest.java +++ b/extended-it/src/test/java/apoc/redis/RedisTest.java @@ -1,7 +1,5 @@ package apoc.redis; -import apoc.redis.Redis; -import apoc.redis.RedisConfig; import apoc.util.TestUtil; import org.apache.commons.lang3.StringUtils; import org.junit.After; @@ -24,7 +22,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import static apoc.util.MapUtil.map; +import static apoc.util.MapUtilExtended.map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; diff --git a/extended-it/src/test/java/apoc/s3/LoadS3Test.java b/extended-it/src/test/java/apoc/s3/LoadS3Test.java index b017bae225..a309ca8ab1 100644 --- a/extended-it/src/test/java/apoc/s3/LoadS3Test.java +++ b/extended-it/src/test/java/apoc/s3/LoadS3Test.java @@ -4,7 +4,7 @@ import apoc.load.LoadJson; import apoc.load.Xml; import apoc.util.TestUtil; -import apoc.util.Util; +import apoc.util.UtilExtended; import apoc.util.s3.S3BaseTest; import apoc.xml.XmlTestUtils; import org.junit.Assert; @@ -22,7 +22,7 @@ import static apoc.ApocConfig.apocConfig; import static apoc.load.LoadCsvTest.assertRow; import static apoc.util.ExtendedITUtil.EXTENDED_PATH; -import static apoc.util.MapUtil.map; +import static apoc.util.MapUtilExtended.map; import static apoc.util.TestUtil.testCall; import static apoc.util.TestUtil.testResult; import static java.util.Arrays.asList; @@ -73,7 +73,7 @@ public void testLoadCsvS3() { String url = s3Container.putFile(EXTENDED_PATH + "src/test/resources/xml/books.xml"); url = removeRegionFromUrl(url); - testCall(db, "CALL apoc.load.xml($url,'/catalog/book[title=\"Maeve Ascendant\"]/.',{failOnError:false}) yield value as result", Util.map("url", url), (r) -> { + testCall(db, "CALL apoc.load.xml($url,'/catalog/book[title=\"Maeve Ascendant\"]/.',{failOnError:false}) yield value as result", UtilExtended.map("url", url), (r) -> { Object value = Iterables.single(r.values()); Assert.assertEquals(XmlTestUtils.XML_XPATH_AS_NESTED_MAP, value); }); diff --git a/extended-it/src/test/java/apoc/vectordb/ChromaDbTest.java b/extended-it/src/test/java/apoc/vectordb/ChromaDbTest.java index 2ceede5c3f..94de7b6d93 100644 --- a/extended-it/src/test/java/apoc/vectordb/ChromaDbTest.java +++ b/extended-it/src/test/java/apoc/vectordb/ChromaDbTest.java @@ -20,7 +20,7 @@ import static apoc.ml.Prompt.API_KEY_CONF; import static apoc.ml.RestAPIConfig.HEADERS_KEY; import static apoc.util.ExtendedTestUtil.assertFails; -import static apoc.util.MapUtil.map; +import static apoc.util.MapUtilExtended.map; import static apoc.util.TestUtil.testCall; import static apoc.util.TestUtil.testResult; import static apoc.vectordb.VectorDbHandler.Type.CHROMA; diff --git a/extended-it/src/test/java/apoc/vectordb/MilvusTest.java b/extended-it/src/test/java/apoc/vectordb/MilvusTest.java index d120da66ef..2137764306 100644 --- a/extended-it/src/test/java/apoc/vectordb/MilvusTest.java +++ b/extended-it/src/test/java/apoc/vectordb/MilvusTest.java @@ -2,7 +2,7 @@ import apoc.ml.Prompt; import apoc.util.TestUtil; -import apoc.util.Util; +import apoc.util.UtilExtended; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -19,7 +19,7 @@ import static apoc.ml.Prompt.API_KEY_CONF; import static apoc.ml.RestAPIConfig.HEADERS_KEY; -import static apoc.util.MapUtil.map; +import static apoc.util.MapUtilExtended.map; import static apoc.util.TestUtil.testCall; import static apoc.util.TestUtil.testResult; import static apoc.vectordb.VectorDbHandler.Type.MILVUS; @@ -173,7 +173,7 @@ public void deleteVector() { assertEquals(200L, value.get("code")); }); - Util.sleep(2000); + UtilExtended.sleep(2000); } @Test diff --git a/extended-it/src/test/java/apoc/vectordb/QdrantTest.java b/extended-it/src/test/java/apoc/vectordb/QdrantTest.java index 3bce11f137..82fc0d5a7b 100644 --- a/extended-it/src/test/java/apoc/vectordb/QdrantTest.java +++ b/extended-it/src/test/java/apoc/vectordb/QdrantTest.java @@ -2,7 +2,7 @@ import apoc.ml.Prompt; import apoc.util.TestUtil; -import apoc.util.Util; +import apoc.util.UtilExtended; import org.junit.AfterClass; import org.junit.Assume; import org.junit.Before; @@ -22,7 +22,7 @@ import static apoc.ml.Prompt.API_KEY_CONF; import static apoc.ml.RestAPIConfig.HEADERS_KEY; import static apoc.util.ExtendedTestUtil.assertFails; -import static apoc.util.MapUtil.map; +import static apoc.util.MapUtilExtended.map; import static apoc.util.TestUtil.testCall; import static apoc.util.TestUtil.testResult; import static apoc.vectordb.VectorDbHandler.Type.QDRANT; @@ -161,8 +161,8 @@ public void getVectorsWithReadOnlyApiKey() { public void writeOperationWithReadOnlyUser() { try { testCall(db, "CALL apoc.vectordb.qdrant.deleteCollection($host, 'test_collection', $conf)", - Util.map("host", HOST, - "conf", Util.map(HEADERS_KEY, READONLY_AUTHORIZATION) + UtilExtended.map("host", HOST, + "conf", UtilExtended.map(HEADERS_KEY, READONLY_AUTHORIZATION) ), r -> fail() ); diff --git a/extended-it/src/test/java/apoc/vectordb/WeaviateTest.java b/extended-it/src/test/java/apoc/vectordb/WeaviateTest.java index 5b3aed4d27..6278c4f047 100644 --- a/extended-it/src/test/java/apoc/vectordb/WeaviateTest.java +++ b/extended-it/src/test/java/apoc/vectordb/WeaviateTest.java @@ -1,10 +1,9 @@ package apoc.vectordb; import apoc.ml.Prompt; -import apoc.util.MapUtil; +import apoc.util.MapUtilExtended; import apoc.util.TestUtil; import org.junit.AfterClass; -import org.junit.Assume; import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; @@ -26,7 +25,7 @@ import static apoc.util.TestUtil.testCall; import static apoc.util.TestUtil.testCallEmpty; import static apoc.util.TestUtil.testResult; -import static apoc.util.Util.map; +import static apoc.util.UtilExtended.map; import static apoc.vectordb.VectorDbHandler.Type.WEAVIATE; import static apoc.vectordb.VectorDbTestUtil.EntityType.FALSE; import static apoc.vectordb.VectorDbTestUtil.EntityType.NODE; @@ -97,7 +96,7 @@ public static void setUp() throws Exception { TestUtil.registerProcedure(db, Weaviate.class, VectorDb.class, Prompt.class); testCall(db, "CALL apoc.vectordb.weaviate.createCollection($host, 'TestCollection', 'cosine', 4, $conf)", - MapUtil.map("host", HOST, "conf", ADMIN_HEADER_CONF), + MapUtilExtended.map("host", HOST, "conf", ADMIN_HEADER_CONF), r -> { Map value = (Map) r.get("value"); assertEquals("TestCollection", value.get("class")); @@ -113,7 +112,7 @@ public static void setUp() throws Exception { ], $conf) """, - MapUtil.map("host", HOST, "id1", ID_1, "id2", ID_2, "conf", ADMIN_HEADER_CONF), + MapUtilExtended.map("host", HOST, "id1", ID_1, "id2", ID_2, "conf", ADMIN_HEADER_CONF), r -> { ResourceIterator values = r.columnAs("value"); assertEquals(COLLECTION_NAME, values.next().get("class")); @@ -137,7 +136,7 @@ public static void setUp() throws Exception { @AfterClass public static void tearDown() throws Exception { testCallEmpty(db, "CALL apoc.vectordb.weaviate.deleteCollection($host, $collectionName, $conf)", - MapUtil.map("host", HOST, "collectionName", COLLECTION_NAME, "conf", ADMIN_HEADER_CONF) + MapUtilExtended.map("host", HOST, "collectionName", COLLECTION_NAME, "conf", ADMIN_HEADER_CONF) ); WEAVIATE_CONTAINER.stop(); @@ -364,9 +363,9 @@ public void getVectorsWithCreateNodeUsingExistingNode() { db.executeTransactionally("CREATE (:Test {myId: 'one'}), (:Test {myId: 'two'})"); - Map conf = MapUtil.map(ALL_RESULTS_KEY, true, + Map conf = MapUtilExtended.map(ALL_RESULTS_KEY, true, HEADERS_KEY, ADMIN_AUTHORIZATION, - MAPPING_KEY, MapUtil.map(EMBEDDING_KEY, "vect", + MAPPING_KEY, MapUtilExtended.map(EMBEDDING_KEY, "vect", NODE_LABEL, "Test", ENTITY_KEY, "myId", METADATA_KEY, "foo")); @@ -391,9 +390,9 @@ public void getVectorsWithCreateNodeUsingExistingNode() { public void getReadOnlyVectorsWithMapping() { db.executeTransactionally("CREATE (:Test {readID: 'one'}), (:Test {readID: 'two'})"); - Map conf = MapUtil.map(ALL_RESULTS_KEY, true, + Map conf = MapUtilExtended.map(ALL_RESULTS_KEY, true, HEADERS_KEY, READONLY_AUTHORIZATION, - MAPPING_KEY, MapUtil.map( + MAPPING_KEY, MapUtilExtended.map( NODE_LABEL, "Test", ENTITY_KEY, "readID", METADATA_KEY, "foo") @@ -401,7 +400,7 @@ public void getReadOnlyVectorsWithMapping() { testResult(db, "CALL apoc.vectordb.weaviate.get($host, 'TestCollection', [$id1, $id2], $conf) " + "YIELD vector, id, metadata, node RETURN * ORDER BY id", - MapUtil.map("host", HOST, "id1", ID_1, "id2", ID_2, "conf", conf), + MapUtilExtended.map("host", HOST, "id1", ID_1, "id2", ID_2, "conf", conf), r -> assertReadOnlyProcWithMappingResults(r, "node") ); } @@ -439,10 +438,10 @@ MAPPING_KEY, map(EMBEDDING_KEY, "vect", public void queryReadOnlyVectorsWithMapping() { db.executeTransactionally("CREATE (:Start)-[:TEST {readID: 'one'}]->(:End), (:Start)-[:TEST {readID: 'two'}]->(:End)"); - Map conf = MapUtil.map(ALL_RESULTS_KEY, true, + Map conf = MapUtilExtended.map(ALL_RESULTS_KEY, true, FIELDS_KEY, FIELDS, HEADERS_KEY, READONLY_AUTHORIZATION, - MAPPING_KEY, MapUtil.map( + MAPPING_KEY, MapUtilExtended.map( REL_TYPE, "TEST", ENTITY_KEY, "readID", METADATA_KEY, "foo") @@ -450,7 +449,7 @@ public void queryReadOnlyVectorsWithMapping() { testResult(db, "CALL apoc.vectordb.weaviate.query($host, 'TestCollection', [0.2, 0.1, 0.9, 0.7], null, 5, $conf) " + " YIELD score, vector, id, metadata, rel RETURN * ORDER BY id", - MapUtil.map("host", HOST, "conf", conf), + MapUtilExtended.map("host", HOST, "conf", conf), r -> assertReadOnlyProcWithMappingResults(r, "rel") ); } @@ -548,11 +547,11 @@ public void queryVectorsWithSystemDbStorage() { public void queryVectorsWithRag() { String openAIKey = ragSetup(db); - Map conf = MapUtil.map( + Map conf = MapUtilExtended.map( FIELDS_KEY, FIELDS, ALL_RESULTS_KEY, true, HEADERS_KEY, READONLY_AUTHORIZATION, - MAPPING_KEY, MapUtil.map(EMBEDDING_KEY, "vect", + MAPPING_KEY, MapUtilExtended.map(EMBEDDING_KEY, "vect", NODE_LABEL, "Rag", ENTITY_KEY, "readID", METADATA_KEY, "foo") @@ -566,11 +565,11 @@ WITH collect(node) as paths RETURN value """ , - MapUtil.map( + MapUtilExtended.map( "host", HOST, "id1", ID_1, "conf", conf, - "confPrompt", MapUtil.map(API_KEY_CONF, openAIKey), + "confPrompt", MapUtilExtended.map(API_KEY_CONF, openAIKey), "attributes", List.of("city", "foo") ), VectorDbTestUtil::assertRagWithVectors); diff --git a/extended-it/src/test/java/apoc/xml/XmlTestUtils.java b/extended-it/src/test/java/apoc/xml/XmlTestUtils.java new file mode 100644 index 0000000000..408b086dd4 --- /dev/null +++ b/extended-it/src/test/java/apoc/xml/XmlTestUtils.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package apoc.xml; + +import java.util.Arrays; +import java.util.Map; + +import static apoc.util.MapUtil.map; +import static java.util.Arrays.asList; + +public class XmlTestUtils { + + public static final Map XML_XPATH_AS_NESTED_MAP = map( + "_type", + "book", + "id", + "bk103", + "_children", + Arrays.asList( + map("_type", "author", "_text", "Corets, Eva"), + map("_type", "title", "_text", "Maeve Ascendant"), + map("_type", "genre", "_text", "Fantasy"), + map("_type", "price", "_text", "5.95"), + map("_type", "publish_date", "_text", "2000-11-17"), + map( + "_type", + "description", + "_text", + "After the collapse of a nanotechnology society in England, the young survivors lay the foundation for a new society."))); +} diff --git a/extended/build.gradle b/extended/build.gradle index ab2d688052..4c8e761fde 100644 --- a/extended/build.gradle +++ b/extended/build.gradle @@ -85,8 +85,8 @@ dependencies { } // These will be dependencies packaged with the .jar - implementation project(':common') implementation group: 'com.unboundid', name: 'unboundid-ldapsdk', version: '6.0.11' + api group: 'com.jayway.jsonpath', name: 'json-path', version: '2.9.0' implementation group: 'org.jsoup', name: 'jsoup', version: '1.15.3' implementation group: 'org.apache.commons', name: 'commons-csv', version: '1.10.0', { exclude group: 'org.apache.commons', module: 'commons-io' @@ -100,6 +100,8 @@ dependencies { compileOnly group: 'org.neo4j', name: 'neo4j', version: neo4jVersionEffective // same version as the one included in neo4j `lib` compileOnly group: 'org.neo4j.driver', name: 'neo4j-java-driver', version: '5.20.0' + compileOnly group: 'com.google.cloud', name: 'google-cloud-storage', version: '2.26.1' + compileOnly group: 'org.apache.commons', name: 'commons-lang3', version: '3.14.0' compileOnly group: 'org.apache.poi', name: 'poi', version: '5.1.0', { exclude group: 'org.apache.commons', module: 'commons-collections4' diff --git a/extended/src/main/java/apoc/ApocGlobalComponentsService.java b/extended/src/main/java/apoc/ApocGlobalComponentsService.java new file mode 100644 index 0000000000..ed9e262020 --- /dev/null +++ b/extended/src/main/java/apoc/ApocGlobalComponentsService.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package apoc; + +import org.neo4j.annotations.service.Service; +import org.neo4j.kernel.availability.AvailabilityListener; +import org.neo4j.kernel.internal.GraphDatabaseAPI; +import org.neo4j.kernel.lifecycle.Lifecycle; + +import java.util.Collection; +import java.util.Map; + +@Service +public interface ApocGlobalComponentsService { + Map getServices(GraphDatabaseAPI db, ExtendedApocExtensionFactory.Dependencies dependencies); + + Collection getContextClasses(); + + Iterable getListeners(GraphDatabaseAPI db, ExtendedApocExtensionFactory.Dependencies dependencies); +} diff --git a/extended/src/main/java/apoc/ExtendedApocConfig.java b/extended/src/main/java/apoc/ExtendedApocConfig.java index f5249cda37..4563c52974 100644 --- a/extended/src/main/java/apoc/ExtendedApocConfig.java +++ b/extended/src/main/java/apoc/ExtendedApocConfig.java @@ -1,12 +1,18 @@ package apoc; -import static apoc.ApocConfig.SUN_JAVA_COMMAND; - +import apoc.export.util.ExportConfigExtended; +import apoc.util.FileUtilsExtended; import apoc.util.SimpleRateLimiter; import java.io.File; +import java.io.IOException; +import java.net.URL; import java.time.Duration; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.stream.Stream; @@ -19,13 +25,34 @@ import org.apache.commons.configuration2.ex.ConversionException; import org.apache.commons.configuration2.io.FileHandler; import org.apache.commons.configuration2.tree.OverrideCombiner; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.configuration.GraphDatabaseSettings; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.graphdb.security.URLAccessChecker; import org.neo4j.kernel.api.procedure.GlobalProcedures; import org.neo4j.kernel.lifecycle.LifecycleAdapter; import org.neo4j.logging.Log; import org.neo4j.logging.internal.LogService; +import org.neo4j.util.Preconditions; + +import static java.lang.String.format; +import static org.neo4j.configuration.BootloaderSettings.lib_directory; +import static org.neo4j.configuration.BootloaderSettings.run_directory; +import static org.neo4j.configuration.GraphDatabaseSettings.SYSTEM_DATABASE_NAME; +import static org.neo4j.configuration.GraphDatabaseSettings.data_directory; +import static org.neo4j.configuration.GraphDatabaseSettings.load_csv_file_url_root; +import static org.neo4j.configuration.GraphDatabaseSettings.logs_directory; +import static org.neo4j.configuration.GraphDatabaseSettings.neo4j_home; +import static org.neo4j.configuration.GraphDatabaseSettings.plugin_dir; +import static org.neo4j.configuration.GraphDatabaseSettings.transaction_logs_root_path; +import static org.neo4j.internal.helpers.ProcessUtils.executeCommandWithOutput; public class ExtendedApocConfig extends LifecycleAdapter { + public static final String SUN_JAVA_COMMAND = "sun.java.command"; public static final String APOC_TTL_SCHEDULE = "apoc.ttl.schedule"; public static final String APOC_TTL_ENABLED = "apoc.ttl.enabled"; public static final String APOC_TTL_LIMIT = "apoc.ttl.limit"; @@ -44,18 +71,57 @@ public class ExtendedApocConfig extends LifecycleAdapter public static final String APOC_ML_WATSON_URL = "apoc.ml.watson.url"; public static final String APOC_AWS_KEY_ID = "apoc.aws.key.id"; public static final String APOC_AWS_SECRET_KEY = "apoc.aws.secret.key"; + + // From core + public static final String APOC_IMPORT_FILE_ENABLED = "apoc.import.file.enabled"; + public static final String APOC_EXPORT_FILE_ENABLED = "apoc.export.file.enabled"; + public static final String APOC_IMPORT_FILE_USE_NEO4J_CONFIG = "apoc.import.file.use_neo4j_config"; + public static final String APOC_TRIGGER_ENABLED = "apoc.trigger.enabled"; + public static final String APOC_IMPORT_FILE_ALLOW__READ__FROM__FILESYSTEM = + "apoc.import.file.allow_read_from_filesystem"; + public static final String APOC_CONFIG_JOBS_SCHEDULED_NUM_THREADS = "apoc.jobs.scheduled.num_threads"; + public static final String APOC_CONFIG_JOBS_POOL_NUM_THREADS = "apoc.jobs.pool.num_threads"; + public static final String APOC_CONFIG_JOBS_QUEUE_SIZE = "apoc.jobs.queue.size"; + public static final String APOC_CONFIG_INITIALIZER = "apoc.initializer"; + public static final String LOAD_FROM_FILE_ERROR = + "Import from files not enabled, please set apoc.import.file.enabled=true in your apoc.conf"; + public static final String APOC_MAX_DECOMPRESSION_RATIO = "apoc.max.decompression.ratio"; + public static final Integer DEFAULT_MAX_DECOMPRESSION_RATIO = 200; + public enum UuidFormatType { hex, base64 } // These were earlier added via the Neo4j config using the ApocSettings.java class private static final Map configDefaultValues = Map.of( + APOC_EXPORT_FILE_ENABLED, false, + APOC_IMPORT_FILE_ENABLED, false, + APOC_IMPORT_FILE_USE_NEO4J_CONFIG, true, + APOC_TRIGGER_ENABLED, false, APOC_TTL_SCHEDULE, Duration.ofMinutes(1), APOC_TTL_ENABLED, false, APOC_TTL_LIMIT, 1000L, APOC_UUID_ENABLED, false ); + private static final List> NEO4J_DIRECTORY_CONFIGURATION_SETTING_NAMES = new ArrayList<>(Arrays.asList( + data_directory, + load_csv_file_url_root, + logs_directory, + plugin_dir, + transaction_logs_root_path, + run_directory, + lib_directory, + neo4j_home)); + + public static final String CONFIG_DIR = "config-dir="; + public static final String EXPORT_NOT_ENABLED_ERROR = + "Export to files not enabled, please set apoc.export.file.enabled=true in your apoc.conf."; + public static final String EXPORT_TO_FILE_ERROR = EXPORT_NOT_ENABLED_ERROR + + "\nOtherwise, if you are running in a cloud environment without filesystem access, use the `{stream:true}` config and null as a 'file' parameter to stream the export back to your client."; + + private final Config neo4jConfig; private final Log log; + private final DatabaseManagementService databaseManagementService; private final String defaultConfigPath; @@ -66,20 +132,36 @@ public enum UuidFormatType { hex, base64 } private ExtendedApocConfig.LoggingType loggingType; private SimpleRateLimiter rateLimiter; + private GraphDatabaseService systemDb; + private boolean expandCommands; + private Duration commandEvaluationTimeout; + private File apocConfFile; /** * keep track if this instance is already initialized so dependent class can wait if needed */ private boolean initialized = false; - public static final String CONFIG_DIR = "config-dir="; - - public ExtendedApocConfig(LogService log, GlobalProcedures globalProceduresRegistry, String defaultConfigPath) { - this.log = log.getInternalLog(ApocConfig.class); + public ExtendedApocConfig( + Config neo4jConfig, + LogService log, + GlobalProcedures globalProceduresRegistry, + String defaultConfigPath, + DatabaseManagementService databaseManagementService) { + this.neo4jConfig = neo4jConfig; + this.commandEvaluationTimeout = + neo4jConfig.get(GraphDatabaseInternalSettings.config_command_evaluation_timeout); + if (this.commandEvaluationTimeout == null) { + this.commandEvaluationTimeout = + GraphDatabaseInternalSettings.config_command_evaluation_timeout.defaultValue(); + } + this.expandCommands = neo4jConfig.expandCommands(); + this.log = log.getInternalLog(ExtendedApocConfig.class); this.defaultConfigPath = defaultConfigPath; theInstance = this; + this.databaseManagementService = databaseManagementService; - // expose this config instance via `@Context ApocConfig config` - globalProceduresRegistry.registerComponent((Class) getClass(), ctx -> this, true); + // expose this config instance via `@Context ExtendedApocConfig config` + globalProceduresRegistry.registerComponent(ExtendedApocConfig.class, ctx -> this, true); this.log.info("successfully registered ExtendedApocConfig for @Context"); } @@ -91,10 +173,30 @@ public void init() { System.setProperty("NEO4J_CONF", neo4jConfFolder); log.info("system property NEO4J_CONF set to %s", neo4jConfFolder); File apocConfFile = new File(neo4jConfFolder + "/apoc.conf"); + // Command Expansion required check from Neo4j + if (apocConfFile.exists() && this.expandCommands) { + Config.Builder.validateFilePermissionForCommandExpansion(List.of(apocConfFile.toPath())); + } + loadConfiguration(apocConfFile); initialized = true; } + private String evaluateIfCommand(String settingName, String entry) { + if (Config.isCommand(entry)) { + Preconditions.checkArgument( + expandCommands, + format( + "%s is a command, but config is not explicitly told to expand it. (Missing --expand-commands argument?)", + entry)); + String str = entry.trim(); + String command = str.substring(2, str.length() - 1); + log.info("Executing external script to retrieve value of setting " + settingName); + return executeCommandWithOutput(command, commandEvaluationTimeout); + } + return entry; + } + public boolean isInitialized() { return initialized; } @@ -131,6 +233,13 @@ protected void loadConfiguration(File apocConfFile) { try { config = setupConfigurations(apocConfFile); + // Command Expansion if needed + config.getKeys() + .forEachRemaining(configKey -> config.setProperty( + configKey, + evaluateIfCommand( + configKey, config.getProperty(configKey).toString()))); + // set config settings not explicitly set in apoc.conf to their default value configDefaultValues.forEach((k,v) -> { if (!config.containsKey(k)) @@ -139,6 +248,29 @@ protected void loadConfiguration(File apocConfFile) { log.info("setting APOC config to default value: " + k + "=" + v); } }); + + for (Setting s : NEO4J_DIRECTORY_CONFIGURATION_SETTING_NAMES) { + Object value = neo4jConfig.get(s); + if (value != null) { + config.setProperty(s.name(), value.toString()); + } + } + + if (!config.containsKey(APOC_MAX_DECOMPRESSION_RATIO)) { + config.setProperty(APOC_MAX_DECOMPRESSION_RATIO, DEFAULT_MAX_DECOMPRESSION_RATIO); + } + if (config.getInt(APOC_MAX_DECOMPRESSION_RATIO) == 0) { + throw new IllegalArgumentException( + format("value 0 is not allowed for the config option %s", APOC_MAX_DECOMPRESSION_RATIO)); + } + + boolean allowFileUrls = neo4jConfig.get(GraphDatabaseSettings.allow_file_urls); + config.setProperty(APOC_IMPORT_FILE_ALLOW__READ__FROM__FILESYSTEM, allowFileUrls); + + // todo - evaluate default timezone here [maybe is reusable], otherwise through db.execute('CALL + // dbms.listConfig()') + final Setting db_temporal_timezone = GraphDatabaseSettings.db_temporal_timezone; + config.setProperty(db_temporal_timezone.name(), neo4jConfig.get(db_temporal_timezone)); initLogging(); } catch ( ConfigurationException e) { @@ -164,7 +296,7 @@ private static Configuration setupConfigurations(File propertyFile) throws Confi return combined; } - protected Configuration getConfig() { + public Configuration getConfig() { return config; } @@ -207,6 +339,49 @@ public boolean containsKey(String key) { return config.containsKey(key); } + public boolean getBoolean(String key) { + return getConfig().getBoolean(key); + } + + public void checkReadAllowed(String url, URLAccessChecker urlAccessChecker) throws IOException { + if (FileUtilsExtended.isFile(url)) { + isImportFileEnabled(); + } else { + checkAllowedUrlAndPinToIP(url, urlAccessChecker); + } + } + + // added because with binary file there isn't an url + public void isImportFileEnabled() { + if (!config.getBoolean(APOC_IMPORT_FILE_ENABLED)) { + throw new RuntimeException(LOAD_FROM_FILE_ERROR); + } + } + + public URL checkAllowedUrlAndPinToIP(String url, URLAccessChecker urlAccessChecker) throws IOException { + try { + URL parsedUrl = new URL(url); + return urlAccessChecker.checkURL(parsedUrl); + } catch (Exception e) { + throw new IOException(e); + } + } + + public boolean isImportFolderConfigured() { + // in case we're test database import path is TestDatabaseManagementServiceBuilder.EPHEMERAL_PATH + + String importFolder = getImportDir(); + if (importFolder == null) { + return false; + } else { + return !"/target/test data/neo4j".equals(importFolder); + } + } + + public String getImportDir() { + return extendedApocConfig().getString("server.directories.import"); + } + public boolean getBoolean(String key, boolean defaultValue) { return getConfig().getBoolean(key, defaultValue); } @@ -223,8 +398,12 @@ public > T getEnumProperty(String key, Class cls, T default return defaultValue; } } + + public void setProperty(String key, Object value) { + getConfig().setProperty(key, value); + } - private int getInt(String key, int defaultValue) { + public int getInt(String key, int defaultValue) { try { return config.getInt(key, defaultValue); } catch ( ConversionException e) { @@ -236,4 +415,30 @@ private int getInt(String key, int defaultValue) { } } } + + + // Methods brought over from Core Config + + public String getString(String key) { + return getConfig().getString(key); + } + + public void checkWriteAllowed(ExportConfigExtended exportConfig, String fileName) { + if (!config.getBoolean(APOC_EXPORT_FILE_ENABLED)) { + if (exportConfig == null || (fileName != null && !fileName.isEmpty()) || !exportConfig.streamStatements()) { + throw new RuntimeException(EXPORT_TO_FILE_ERROR); + } + } + } + + public GraphDatabaseService getSystemDb() { + if (systemDb == null) { + try { + systemDb = databaseManagementService.database(SYSTEM_DATABASE_NAME); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + return systemDb; + } } diff --git a/extended/src/main/java/apoc/ExtendedApocConfigExtensionFactory.java b/extended/src/main/java/apoc/ExtendedApocConfigExtensionFactory.java index e1dcec241d..d3f9df32cc 100644 --- a/extended/src/main/java/apoc/ExtendedApocConfigExtensionFactory.java +++ b/extended/src/main/java/apoc/ExtendedApocConfigExtensionFactory.java @@ -2,6 +2,7 @@ import org.neo4j.annotations.service.ServiceProvider; import org.neo4j.configuration.Config; +import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.kernel.api.procedure.GlobalProcedures; import org.neo4j.kernel.extension.ExtensionFactory; import org.neo4j.kernel.extension.ExtensionType; @@ -16,9 +17,12 @@ public class ExtendedApocConfigExtensionFactory extends ExtensionFactory { + + public ExtendedApocExtensionFactory() { + super(ExtensionType.DATABASE, "APOC_EXTENDED"); + } + + public interface Dependencies { + GraphDatabaseAPI graphdatabaseAPI(); + + JobScheduler scheduler(); + + LogService log(); + + AvailabilityGuard availabilityGuard(); + + DatabaseManagementService databaseManagementService(); + + ExtendedApocConfig extendedApocConfig(); + + DatabaseEventListeners databaseEventListeners(); + + GlobalProcedures globalProceduresRegistry(); + + ExtendedRegisterComponentFactory.RegisterComponentLifecycle registerComponentLifecycle(); + + PoolsExtended pools(); + } + + @Override + public Lifecycle newInstance(ExtensionContext context, Dependencies dependencies) { + GraphDatabaseAPI db = dependencies.graphdatabaseAPI(); + LogService log = dependencies.log(); + return new ApocLifecycle(log, db, dependencies); + } + + public static class ApocLifecycle extends LifecycleAdapter { + private final Log userLog; + private final GraphDatabaseAPI db; + private final Dependencies dependencies; + private final Map services = new HashMap<>(); + private final Collection apocGlobalComponents; + private final Collection registeredListeners = new ArrayList<>(); + + public ApocLifecycle(LogService log, GraphDatabaseAPI db, Dependencies dependencies) { + this.db = db; + this.dependencies = dependencies; + this.userLog = log.getUserLog(ExtendedApocExtensionFactory.class); + this.apocGlobalComponents = Services.loadAll(ApocGlobalComponentsService.class); + } + + public static void withNonSystemDatabase(GraphDatabaseService db, Consumer consumer) { + if (!SYSTEM_DATABASE_NAME.equals(db.databaseName())) { + consumer.accept(null); + } + } + + @Override + public void init() { + withNonSystemDatabase(db, aVoid -> { + for (ApocGlobalComponentsService c : apocGlobalComponents) { + services.putAll(c.getServices(db, dependencies)); + } + + String databaseName = db.databaseName(); + services.values().forEach(lifecycle -> dependencies + .registerComponentLifecycle() + .addResolver(databaseName, lifecycle.getClass(), lifecycle)); + }); + } + + @Override + public void start() { + withNonSystemDatabase(db, aVoid -> { + services.forEach((key, value) -> { + try { + value.start(); + } catch (Exception e) { + userLog.error("failed to start service " + key, e); + } + }); + }); + + AvailabilityGuard availabilityGuard = dependencies.availabilityGuard(); + for (ApocGlobalComponentsService c : apocGlobalComponents) { + for (AvailabilityListener listener : c.getListeners(db, dependencies)) { + registeredListeners.add(listener); + availabilityGuard.addListener(listener); + } + } + } + + @Override + public void stop() { + withNonSystemDatabase(db, aVoid -> { + services.forEach((key, value) -> { + try { + value.stop(); + } catch (Exception e) { + userLog.error("failed to stop service " + key, e); + } + }); + }); + + AvailabilityGuard availabilityGuard = dependencies.availabilityGuard(); + registeredListeners.forEach(availabilityGuard::removeListener); + registeredListeners.clear(); + } + + public Collection getRegisteredListeners() { + return registeredListeners; + } + } +} diff --git a/extended/src/main/java/apoc/ExtendedApocGlobalComponents.java b/extended/src/main/java/apoc/ExtendedApocGlobalComponents.java index 3ef25bb5b3..fd898e3438 100644 --- a/extended/src/main/java/apoc/ExtendedApocGlobalComponents.java +++ b/extended/src/main/java/apoc/ExtendedApocGlobalComponents.java @@ -20,18 +20,18 @@ import java.util.concurrent.ConcurrentHashMap; @ServiceProvider -public class ExtendedApocGlobalComponents implements ApocGlobalComponents { +public class ExtendedApocGlobalComponents implements ApocGlobalComponentsService { private final Map cypherProcedureHandlers = new ConcurrentHashMap<>(); @Override - public Map getServices(GraphDatabaseAPI db, ApocExtensionFactory.Dependencies dependencies) { + public Map getServices(GraphDatabaseAPI db, ExtendedApocExtensionFactory.Dependencies dependencies) { CypherProceduresHandler cypherProcedureHandler = new CypherProceduresHandler( db, dependencies.scheduler(), - dependencies.apocConfig(), + dependencies.extendedApocConfig(), dependencies.log().getUserLog(CypherProcedures.class), dependencies.globalProceduresRegistry() ); @@ -47,7 +47,7 @@ public Map getServices(GraphDatabaseAPI db, ApocExtensionFact "uuid", new UuidHandler(db, dependencies.databaseManagementService(), dependencies.log().getUserLog(Uuid.class), - dependencies.apocConfig(), + dependencies.extendedApocConfig(), dependencies.scheduler(), dependencies.pools()), @@ -65,7 +65,7 @@ public Collection getContextClasses() { } @Override - public Iterable getListeners(GraphDatabaseAPI db, ApocExtensionFactory.Dependencies dependencies) { + public Iterable getListeners(GraphDatabaseAPI db, ExtendedApocExtensionFactory.Dependencies dependencies) { CypherProceduresHandler cypherProceduresHandler = cypherProcedureHandlers.get(db); return cypherProceduresHandler==null ? Collections.emptyList() : Collections.singleton(cypherProceduresHandler); } diff --git a/extended/src/main/java/apoc/ExtendedRegisterComponentFactory.java b/extended/src/main/java/apoc/ExtendedRegisterComponentFactory.java index 2c2707a1b8..88ecb36de1 100644 --- a/extended/src/main/java/apoc/ExtendedRegisterComponentFactory.java +++ b/extended/src/main/java/apoc/ExtendedRegisterComponentFactory.java @@ -1,5 +1,6 @@ package apoc; +import org.neo4j.annotations.service.ServiceProvider; import org.neo4j.kernel.api.procedure.GlobalProcedures; import org.neo4j.kernel.extension.ExtensionFactory; import org.neo4j.kernel.extension.ExtensionType; @@ -16,6 +17,7 @@ /** * NOTE: this is a GLOBAL component, so only once per DBMS */ +@ServiceProvider public class ExtendedRegisterComponentFactory extends ExtensionFactory { private Log log; @@ -54,7 +56,7 @@ public Map> getResolvers() { @Override public void init() throws Exception { - for (ExtendedApocGlobalComponents c: Services.loadAll(ExtendedApocGlobalComponents.class)) { + for (ApocGlobalComponentsService c: Services.loadAll(ApocGlobalComponentsService.class)) { for (Class clazz: c.getContextClasses()) { resolvers.put(clazz, new ConcurrentHashMap<>()); } diff --git a/extended/src/main/java/apoc/ExtendedSystemLabels.java b/extended/src/main/java/apoc/ExtendedSystemLabels.java index a50c89d442..66fcdb7fef 100644 --- a/extended/src/main/java/apoc/ExtendedSystemLabels.java +++ b/extended/src/main/java/apoc/ExtendedSystemLabels.java @@ -2,13 +2,14 @@ import org.neo4j.graphdb.Label; -public enum ExtendedSystemLabels implements Label +public enum ExtendedSystemLabels implements Label { ApocCypherProcedures, ApocCypherProceduresMeta, Procedure, Function, ApocUuid, + ApocTrigger, ApocUuidMeta, DataVirtualizationCatalog, VectorDb diff --git a/extended/src/main/java/apoc/ExtendedSystemPropertyKeys.java b/extended/src/main/java/apoc/ExtendedSystemPropertyKeys.java index 6d134e2c5c..b3d8445e18 100644 --- a/extended/src/main/java/apoc/ExtendedSystemPropertyKeys.java +++ b/extended/src/main/java/apoc/ExtendedSystemPropertyKeys.java @@ -2,6 +2,12 @@ public enum ExtendedSystemPropertyKeys { + database, + + // cypher stored procedures/functions + lastUpdated, + statement, + // cypher stored procedures/functions inputs, description, diff --git a/extended/src/main/java/apoc/JdbcRegistererInitFactory.java b/extended/src/main/java/apoc/JdbcRegistererInitFactory.java index e797c954b6..3e87077be9 100644 --- a/extended/src/main/java/apoc/JdbcRegistererInitFactory.java +++ b/extended/src/main/java/apoc/JdbcRegistererInitFactory.java @@ -1,8 +1,8 @@ package apoc; import apoc.load.Jdbc; -import apoc.util.Util; -import apoc.util.collection.Iterators; +import apoc.util.UtilExtended; +import apoc.util.collection.IteratorsExtended; import org.neo4j.annotations.service.ServiceProvider; import org.neo4j.kernel.extension.ExtensionFactory; import org.neo4j.kernel.extension.ExtensionType; @@ -27,12 +27,12 @@ public Lifecycle newInstance(ExtensionContext context, Dependencies dependencies @Override public void init() throws Exception { // we need to await initialization of ExtendedApocConfig. Unfortunately Neo4j's internal service loading tooling does *not* honor the order of service loader META-INF/services files. - Util.newDaemonThread(() -> { + UtilExtended.newDaemonThread(() -> { ExtendedApocConfig extendedApocConfig = dependencies.extendedApocConfig(); while (!extendedApocConfig.isInitialized()) { - Util.sleep(10); + UtilExtended.sleep(10); } - Iterators.stream(extendedApocConfig.getKeys("apoc.jdbc")) + IteratorsExtended.stream(extendedApocConfig.getKeys("apoc.jdbc")) .filter(k -> k.endsWith("driver")) .forEach( Jdbc::loadDriver ); }).start(); diff --git a/extended/src/main/java/apoc/PoolExtensionFactoryExtended.java b/extended/src/main/java/apoc/PoolExtensionFactoryExtended.java new file mode 100644 index 0000000000..910dce2508 --- /dev/null +++ b/extended/src/main/java/apoc/PoolExtensionFactoryExtended.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package apoc; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.kernel.api.procedure.GlobalProcedures; +import org.neo4j.kernel.extension.ExtensionFactory; +import org.neo4j.kernel.extension.ExtensionType; +import org.neo4j.kernel.extension.context.ExtensionContext; +import org.neo4j.kernel.lifecycle.Lifecycle; +import org.neo4j.logging.internal.LogService; + +@ServiceProvider +@SuppressWarnings("unused") +public class PoolExtensionFactoryExtended extends ExtensionFactory { + + public PoolExtensionFactoryExtended() { + super(ExtensionType.GLOBAL, "APOC_POOLS"); + } + + public interface Dependencies { + GlobalProcedures globalProceduresRegistry(); + + LogService log(); + + ExtendedApocConfig apocConfig(); + } + + @Override + public Lifecycle newInstance(ExtensionContext context, Dependencies dependencies) { + return new PoolsExtended(dependencies.log(), dependencies.globalProceduresRegistry(), dependencies.apocConfig()); + } +} diff --git a/extended/src/main/java/apoc/PoolsExtended.java b/extended/src/main/java/apoc/PoolsExtended.java new file mode 100644 index 0000000000..aa3ece7e9f --- /dev/null +++ b/extended/src/main/java/apoc/PoolsExtended.java @@ -0,0 +1,201 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package apoc; + +import apoc.periodic.PeriodicUtilsExtended; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Transaction; +import org.neo4j.kernel.api.procedure.GlobalProcedures; +import org.neo4j.kernel.lifecycle.LifecycleAdapter; +import org.neo4j.logging.Log; +import org.neo4j.logging.internal.LogService; + +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.FutureTask; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.BiConsumer; +import java.util.stream.Stream; + +public class PoolsExtended extends LifecycleAdapter { + + public static final int DEFAULT_SCHEDULED_THREADS = Runtime.getRuntime().availableProcessors() / 4; + public static final int DEFAULT_POOL_THREADS = Runtime.getRuntime().availableProcessors() * 2; + private final Log log; + private final ExtendedApocConfig apocConfig; + + private ExecutorService singleExecutorService; + private ScheduledExecutorService scheduledExecutorService; + private ExecutorService defaultExecutorService; + + private final Map jobList = new ConcurrentHashMap<>(); + + public PoolsExtended(LogService log, GlobalProcedures globalProceduresRegistry, ExtendedApocConfig apocConfig) { + + this.log = log.getInternalLog(PoolsExtended.class); + this.apocConfig = apocConfig; + + // expose this config instance via `@Context ApocConfig config` + globalProceduresRegistry.registerComponent((Class) getClass(), ctx -> this, true); + this.log.info("successfully registered PoolsExtended for @Context"); + } + + @Override + public void init() { + + int threads = + Math.max(1, apocConfig.getInt(ExtendedApocConfig.APOC_CONFIG_JOBS_POOL_NUM_THREADS, DEFAULT_POOL_THREADS)); + + int queueSize = Math.max(1, apocConfig.getInt(ExtendedApocConfig.APOC_CONFIG_JOBS_QUEUE_SIZE, threads * 5)); + + // ensure we use daemon threads everywhere + ThreadFactory threadFactory = r -> { + Thread t = Executors.defaultThreadFactory().newThread(r); + t.setDaemon(true); + return t; + }; + this.singleExecutorService = new ThreadPoolExecutor( + 1, + 1, + 0L, + TimeUnit.SECONDS, + new ArrayBlockingQueue<>(queueSize), + threadFactory, + new CallerBlocksPolicy()); + + this.defaultExecutorService = new ThreadPoolExecutor( + threads / 2, + threads, + 30L, + TimeUnit.SECONDS, + new ArrayBlockingQueue<>(queueSize), + threadFactory, + new CallerBlocksPolicy()); + + this.scheduledExecutorService = Executors.newScheduledThreadPool( + Math.max( + 1, + apocConfig.getInt( + ExtendedApocConfig.APOC_CONFIG_JOBS_SCHEDULED_NUM_THREADS, DEFAULT_SCHEDULED_THREADS)), + threadFactory); + + scheduledExecutorService.scheduleAtFixedRate( + () -> { + for (Iterator> it = + jobList.entrySet().iterator(); + it.hasNext(); ) { + Map.Entry entry = it.next(); + if (entry.getValue().isDone() || entry.getValue().isCancelled()) it.remove(); + } + }, + 10, + 10, + TimeUnit.SECONDS); + } + + @Override + public void shutdown() { + Stream.of(singleExecutorService, defaultExecutorService, scheduledExecutorService) + .forEach(service -> { + try { + service.shutdown(); + service.awaitTermination(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + throw new RuntimeException("Shutdown failed to complete with error: " + e.getMessage()); + } + }); + } + + public ExecutorService getSingleExecutorService() { + return singleExecutorService; + } + + public ScheduledExecutorService getScheduledExecutorService() { + return scheduledExecutorService; + } + + public ExecutorService getDefaultExecutorService() { + return defaultExecutorService; + } + + public Map getJobList() { + return jobList; + } + + static class CallerBlocksPolicy implements RejectedExecutionHandler { + @Override + public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { + // Submit again by directly injecting the task into the work queue, waiting if necessary, but also + // periodically checking if the pool has been + // shut down. + FutureTask task = new FutureTask<>(r, null); + BlockingQueue queue = executor.getQueue(); + while (!executor.isShutdown()) { + try { + if (queue.offer(task, 250, TimeUnit.MILLISECONDS)) { + while (!executor.isShutdown()) { + try { + task.get(250, TimeUnit.MILLISECONDS); + return; // Success! + } catch (TimeoutException ignore) { + // This is fine an expected. We just want to check that the executor hasn't been shut + // down. + } + } + } + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + } + } + } + + public Future processBatch(List batch, GraphDatabaseService db, BiConsumer action) { + return defaultExecutorService.submit(() -> { + try (Transaction tx = db.beginTx()) { + batch.forEach(t -> action.accept(tx, t)); + tx.commit(); + } + return null; + }); + } + + public static T force(Future future) throws ExecutionException { + while (true) { + try { + return future.get(); + } catch (InterruptedException e) { + Thread.interrupted(); + } + } + } +} diff --git a/extended/src/main/java/apoc/TTLConfig.java b/extended/src/main/java/apoc/TTLConfig.java index 1c737e7b6d..19c91e7715 100644 --- a/extended/src/main/java/apoc/TTLConfig.java +++ b/extended/src/main/java/apoc/TTLConfig.java @@ -5,11 +5,11 @@ import org.neo4j.kernel.lifecycle.LifecycleAdapter; public class TTLConfig extends LifecycleAdapter { - private final ApocConfig apocConfig; + private final ExtendedApocConfig apocConfig; public static final int DEFAULT_SCHEDULE = 60; private static TTLConfig theInstance; - public TTLConfig(ApocConfig apocConfig, GlobalProcedures globalProceduresRegistry) { + public TTLConfig(ExtendedApocConfig apocConfig, GlobalProcedures globalProceduresRegistry) { this.apocConfig = apocConfig; theInstance = this; globalProceduresRegistry.registerComponent((Class) getClass(), ctx -> this, true); diff --git a/extended/src/main/java/apoc/TTLConfigExtensionFactory.java b/extended/src/main/java/apoc/TTLConfigExtensionFactory.java index 8a6e9f54a9..21a8f12352 100644 --- a/extended/src/main/java/apoc/TTLConfigExtensionFactory.java +++ b/extended/src/main/java/apoc/TTLConfigExtensionFactory.java @@ -14,7 +14,7 @@ public class TTLConfigExtensionFactory extends ExtensionFactory { public interface Dependencies { - ApocConfig config(); + ExtendedApocConfig config(); GlobalProcedures globalProceduresRegistry(); } diff --git a/extended/src/main/java/apoc/agg/AggregationExtended.java b/extended/src/main/java/apoc/agg/AggregationExtended.java index 128131c5fe..94e057ece7 100644 --- a/extended/src/main/java/apoc/agg/AggregationExtended.java +++ b/extended/src/main/java/apoc/agg/AggregationExtended.java @@ -1,8 +1,8 @@ package apoc.agg; import apoc.Extended; -import apoc.util.collection.Iterables; -import apoc.util.collection.Iterators; +import apoc.util.collection.IterablesExtended; +import apoc.util.collection.IteratorsExtended; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.Transaction; import org.neo4j.procedure.Context; @@ -29,7 +29,7 @@ public class AggregationExtended { public RowFunction row() { BiPredicate curr = (current, value) -> db.executeTransactionally("RETURN " + value, Map.of("curr", current), - result -> Iterators.singleOrNull(result.columnAs(Iterables.single(result.columns())))); + result -> IteratorsExtended.singleOrNull(result.columnAs(IterablesExtended.single(result.columns())))); return new RowFunction(curr); } diff --git a/extended/src/main/java/apoc/agg/Rollup.java b/extended/src/main/java/apoc/agg/Rollup.java index 4a869861fe..9522611cfb 100644 --- a/extended/src/main/java/apoc/agg/Rollup.java +++ b/extended/src/main/java/apoc/agg/Rollup.java @@ -2,7 +2,7 @@ import apoc.Extended; import apoc.util.ExtendedListUtils; -import apoc.util.Util; +import apoc.util.UtilExtended; import org.neo4j.graphdb.Entity; import org.neo4j.procedure.Description; import org.neo4j.procedure.Name; @@ -67,7 +67,7 @@ public void aggregate( @Name(value = "aggKeys") List aggKeys, @Name(value = "config", defaultValue = "{}") Map config) { - boolean cube = Util.toBoolean(config.get("cube")); + boolean cube = UtilExtended.toBoolean(config.get("cube")); Entity entity = (Entity) value; diff --git a/extended/src/main/java/apoc/algo/PathFindingExtended.java b/extended/src/main/java/apoc/algo/PathFindingExtended.java index f9eb496461..a0774eb0da 100644 --- a/extended/src/main/java/apoc/algo/PathFindingExtended.java +++ b/extended/src/main/java/apoc/algo/PathFindingExtended.java @@ -1,7 +1,7 @@ package apoc.algo; import apoc.Extended; -import apoc.result.WeightedPathResult; +import apoc.result.WeightedPathResultExtended; import org.neo4j.graphalgo.BasicEvaluationContext; import org.neo4j.graphalgo.CommonEvaluators; import org.neo4j.graphalgo.GraphAlgoFactory; @@ -16,7 +16,8 @@ import org.neo4j.procedure.Procedure; import java.util.stream.Stream; -import static apoc.algo.PathFindingUtils.buildPathExpander; + +import static apoc.algo.PathFindingExtendedUtils.buildPathExpander; @Extended public class PathFindingExtended { @@ -30,7 +31,7 @@ public class PathFindingExtended { @Procedure @Description("apoc.algo.aStarWithPoint(startNode, endNode, 'relTypesAndDirs', 'distance','pointProp') - " + "equivalent to apoc.algo.aStar but accept a Point type as a pointProperty instead of Number types as latitude and longitude properties") - public Stream aStarWithPoint( + public Stream aStarWithPoint( @Name("startNode") Node startNode, @Name("endNode") Node endNode, @Name("relationshipTypesAndDirections") String relTypesAndDirs, @@ -41,8 +42,8 @@ public Stream aStarWithPoint( new BasicEvaluationContext(tx, db), buildPathExpander(relTypesAndDirs), CommonEvaluators.doubleCostEvaluator(weightPropertyName), - new PathFindingUtils.GeoEstimateEvaluatorPointCustom(pointPropertyName)); - return WeightedPathResult.streamWeightedPathResult(startNode, endNode, algo); + new PathFindingExtendedUtils.GeoEstimateEvaluatorPointCustomExtended(pointPropertyName)); + return WeightedPathResultExtended.streamWeightedPathResult(startNode, endNode, algo); } } diff --git a/extended/src/main/java/apoc/algo/PathFindingExtendedUtils.java b/extended/src/main/java/apoc/algo/PathFindingExtendedUtils.java new file mode 100644 index 0000000000..deb7959f29 --- /dev/null +++ b/extended/src/main/java/apoc/algo/PathFindingExtendedUtils.java @@ -0,0 +1,95 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package apoc.algo; + +import apoc.path.RelationshipTypeAndDirectionsExtended; +import org.apache.commons.lang3.tuple.Pair; +import org.neo4j.graphalgo.EstimateEvaluator; +import org.neo4j.graphdb.Direction; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.PathExpander; +import org.neo4j.graphdb.PathExpanderBuilder; +import org.neo4j.graphdb.RelationshipType; +import org.neo4j.values.storable.PointValue; + + +public class PathFindingExtendedUtils { + public static class GeoEstimateEvaluatorPointCustomExtended implements EstimateEvaluator { + // -- from org.neo4j.graphalgo.impl.util.GeoEstimateEvaluator + private static final double EARTH_RADIUS = 6371 * 1000; // Meters + private Node cachedGoal; + private final String pointPropertyKey; + private double[] cachedGoalCoordinates; + + public GeoEstimateEvaluatorPointCustomExtended(String pointPropertyKey) { + this.pointPropertyKey = pointPropertyKey; + } + + @Override + public Double getCost(Node node, Node goal) { + double[] nodeCoordinates = getCoordinates(node); + if (cachedGoal == null || !cachedGoal.equals(goal)) { + cachedGoalCoordinates = getCoordinates(goal); + cachedGoal = goal; + } + return distance(nodeCoordinates[0], nodeCoordinates[1], cachedGoalCoordinates[0], cachedGoalCoordinates[1]); + } + + private static double distance(double latitude1, double longitude1, double latitude2, double longitude2) { + latitude1 = Math.toRadians(latitude1); + longitude1 = Math.toRadians(longitude1); + latitude2 = Math.toRadians(latitude2); + longitude2 = Math.toRadians(longitude2); + double cLa1 = Math.cos(latitude1); + double xA = EARTH_RADIUS * cLa1 * Math.cos(longitude1); + double yA = EARTH_RADIUS * cLa1 * Math.sin(longitude1); + double zA = EARTH_RADIUS * Math.sin(latitude1); + double cLa2 = Math.cos(latitude2); + double xB = EARTH_RADIUS * cLa2 * Math.cos(longitude2); + double yB = EARTH_RADIUS * cLa2 * Math.sin(longitude2); + double zB = EARTH_RADIUS * Math.sin(latitude2); + return Math.sqrt((xA - xB) * (xA - xB) + (yA - yB) * (yA - yB) + (zA - zB) * (zA - zB)); + } + // -- end from org.neo4j.graphalgo.impl.util.GeoEstimateEvaluator + + private double[] getCoordinates(Node node) { + return ((PointValue) node.getProperty(pointPropertyKey)).coordinate(); + } + } + + public static PathExpander buildPathExpander(String relationshipsAndDirections) { + PathExpanderBuilder builder = PathExpanderBuilder.empty(); + for (Pair pair : RelationshipTypeAndDirectionsExtended.parse(relationshipsAndDirections)) { + if (pair.getLeft() == null) { + if (pair.getRight() == null) { + builder = PathExpanderBuilder.allTypesAndDirections(); + } else { + builder = PathExpanderBuilder.allTypes(pair.getRight()); + } + } else { + if (pair.getRight() == null) { + builder = builder.add(pair.getLeft()); + } else { + builder = builder.add(pair.getLeft(), pair.getRight()); + } + } + } + return builder.build(); + } +} diff --git a/extended/src/main/java/apoc/bolt/Bolt.java b/extended/src/main/java/apoc/bolt/Bolt.java index c630e72c61..5d26472d12 100644 --- a/extended/src/main/java/apoc/bolt/Bolt.java +++ b/extended/src/main/java/apoc/bolt/Bolt.java @@ -1,7 +1,7 @@ package apoc.bolt; import apoc.Extended; -import apoc.result.RowResult; +import apoc.result.RowResultExtended; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.procedure.Context; import org.neo4j.procedure.Description; @@ -32,23 +32,23 @@ private T withConnection(String url, Map config, Function load(@Name("url") String url, @Name("kernelTransaction") String statement, @Name(value = "params", defaultValue = "{}") Map params, @Name(value = "config", defaultValue = "{}") Map config) throws URISyntaxException { + public Stream load(@Name("url") String url, @Name("kernelTransaction") String statement, @Name(value = "params", defaultValue = "{}") Map params, @Name(value = "config", defaultValue = "{}") Map config) throws URISyntaxException { return withConnection(url, config, conn -> conn.loadFromSession(statement, params)); } @Procedure(value = "apoc.bolt.load.fromLocal", mode = Mode.WRITE) - public Stream fromLocal(@Name("url") String url, - @Name("localStatement") String localStatement, - @Name("remoteStatement") String remoteStatement, - @Name(value = "config", defaultValue = "{}") Map config) throws URISyntaxException { + public Stream fromLocal(@Name("url") String url, + @Name("localStatement") String localStatement, + @Name("remoteStatement") String remoteStatement, + @Name(value = "config", defaultValue = "{}") Map config) throws URISyntaxException { return withConnection(url, config, conn -> conn.loadFromLocal(localStatement, remoteStatement, db)); } @Procedure() @Description("apoc.bolt.execute(url-or-key, kernelTransaction, params, config) - access to other databases via bolt for reads and writes") - public Stream execute(@Name("url") String url, @Name("kernelTransaction") String statement, @Name(value = "params", defaultValue = "{}") Map params, @Name(value = "config", defaultValue = "{}") Map config) throws URISyntaxException { + public Stream execute(@Name("url") String url, @Name("kernelTransaction") String statement, @Name(value = "params", defaultValue = "{}") Map params, @Name(value = "config", defaultValue = "{}") Map config) throws URISyntaxException { Map configuration = new HashMap<>(config); configuration.put("readOnly", false); return load(url, statement, params, configuration); diff --git a/extended/src/main/java/apoc/bolt/BoltConnection.java b/extended/src/main/java/apoc/bolt/BoltConnection.java index 0397c87335..1141977d31 100644 --- a/extended/src/main/java/apoc/bolt/BoltConnection.java +++ b/extended/src/main/java/apoc/bolt/BoltConnection.java @@ -1,11 +1,11 @@ package apoc.bolt; -import apoc.result.RowResult; -import apoc.result.VirtualNode; -import apoc.result.VirtualRelationship; +import apoc.result.RowResultExtended; +import apoc.result.VirtualNodeExtended; +import apoc.result.VirtualRelationshipExtended; import apoc.util.UriResolver; -import apoc.util.Util; -import apoc.util.collection.Iterators; +import apoc.util.UtilExtended; +import apoc.util.collection.IteratorsExtended; import org.apache.commons.lang3.StringUtils; import org.neo4j.driver.Driver; import org.neo4j.driver.GraphDatabase; @@ -37,7 +37,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static apoc.util.MapUtil.map; +import static apoc.util.MapUtilExtended.map; public class BoltConnection { private final BoltConfig config; @@ -55,18 +55,18 @@ public static BoltConnection from(Map config, String url) throws } // methods from Bolt.java - public Stream loadFromSession(String statement, Map params) { + public Stream loadFromSession(String statement, Map params) { return withDriverAndSession(session -> { if (config.isAddStatistics()) { Result statementResult = session.run(statement, params); SummaryCounters counters = statementResult.consume().counters(); - return Stream.of(new RowResult(toMap(counters))); + return Stream.of(new RowResultExtended(toMap(counters))); } else return getRowResultStream(session, params, statement); }); } - public Stream loadFromLocal(String localStatement, String remoteStatement, GraphDatabaseService db) { + public Stream loadFromLocal(String localStatement, String remoteStatement, GraphDatabaseService db) { return withDriverAndSession(session -> { try (org.neo4j.graphdb.Transaction tx = db.beginTx()) { final org.neo4j.graphdb.Result localResult = tx.execute(localStatement, config.getLocalParams()); @@ -74,7 +74,7 @@ public Stream loadFromLocal(String localStatement, String remoteState .map(c -> "$" + c + " AS " + c) .collect(Collectors.joining(", ")) + "\n"; Map nodesCache = new HashMap<>(); - List response = new ArrayList<>(); + List response = new ArrayList<>(); while (localResult.hasNext()) { final Result statementResult; Map row = localResult.next(); @@ -87,7 +87,7 @@ public Stream loadFromLocal(String localStatement, String remoteState statementResult = session.run(withColumns + remoteStatement, row); } if (config.isStreamStatements()) { - response.add(new RowResult(toMap(statementResult.consume().counters()))); + response.add(new RowResultExtended(toMap(statementResult.consume().counters()))); } else { response.addAll( statementResult.stream() @@ -120,8 +120,8 @@ private Stream withTransaction(Session session, Function nodesCache) { - return new RowResult(record.asMap(value -> convertRecursive(value, nodesCache))); + private RowResultExtended buildRowResult(Record record, Map nodesCache) { + return new RowResultExtended(record.asMap(value -> convertRecursive(value, nodesCache))); } private Object convertRecursive(Object entity, Map nodesCache) { @@ -144,12 +144,12 @@ private Object toCollection(Collection entity, Map nodeCache) { return entity.stream().map(elem -> convertRecursive(elem, nodeCache)).collect(Collectors.toList()); } - private Stream getRowResultStream(Session session, Map params, String statement) { + private Stream getRowResultStream(Session session, Map params, String statement) { Map nodesCache = new HashMap<>(); return withTransaction(session, tx -> { ClosedAwareDelegatingIterator iterator = new ClosedAwareDelegatingIterator(tx.run(statement, params)); - return Iterators.stream(iterator).map(record -> buildRowResult(record, nodesCache)); + return IteratorsExtended.stream(iterator).map(record -> buildRowResult(record, nodesCache)); }); } @@ -159,23 +159,23 @@ private Object toNode(Object value, Map nodesCache) { if (config.isVirtual()) { List