Skip to content

Commit 2535676

Browse files
authored
Merge pull request #10798 from brs96/apsp-mem-est
AllPairsShortestPaths memory estimation
2 parents 5baf00f + f0ff4cc commit 2535676

File tree

12 files changed

+202
-13
lines changed

12 files changed

+202
-13
lines changed
Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
/*
2+
* Copyright (c) "Neo4j"
3+
* Neo4j Sweden AB [http://neo4j.com]
4+
*
5+
* This file is part of Neo4j.
6+
*
7+
* Neo4j is free software: you can redistribute it and/or modify
8+
* it under the terms of the GNU General Public License as published by
9+
* the Free Software Foundation, either version 3 of the License, or
10+
* (at your option) any later version.
11+
*
12+
* This program is distributed in the hope that it will be useful,
13+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
14+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15+
* GNU General Public License for more details.
16+
*
17+
* You should have received a copy of the GNU General Public License
18+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
19+
*/
20+
package org.neo4j.gds.allshortestpaths;
21+
22+
import org.neo4j.gds.mem.MemoryEstimateDefinition;
23+
import org.neo4j.gds.mem.MemoryEstimation;
24+
import org.neo4j.gds.mem.MemoryEstimations;
25+
import org.neo4j.gds.mem.Estimate;
26+
import org.neo4j.gds.msbfs.MSBFSMemoryEstimation;
27+
28+
public final class AllShortestPathsMemoryEstimateDefinition implements MemoryEstimateDefinition {
29+
30+
private final boolean hasRelationshipWeightProperty;
31+
32+
public AllShortestPathsMemoryEstimateDefinition(boolean hasRelationshipWeightProperty) {
33+
this.hasRelationshipWeightProperty = hasRelationshipWeightProperty;
34+
}
35+
36+
@Override
37+
public MemoryEstimation memoryEstimation() {
38+
if (hasRelationshipWeightProperty) {
39+
return weightedMemoryEstimation();
40+
} else {
41+
return unweightedMemoryEstimation();
42+
}
43+
}
44+
45+
private MemoryEstimation weightedMemoryEstimation() {
46+
return MemoryEstimations.builder(WeightedAllShortestPaths.class)
47+
.perThread("ShortestPathTask", shortestPathTaskMemoryEstimation())
48+
.build();
49+
}
50+
51+
private MemoryEstimation unweightedMemoryEstimation() {
52+
return MemoryEstimations.builder(MSBFSAllShortestPaths.class)
53+
.add("MSBFS", MSBFSMemoryEstimation.MSBFSWithANPStrategy(0))
54+
.build();
55+
}
56+
57+
private MemoryEstimation shortestPathTaskMemoryEstimation() {
58+
return MemoryEstimations.builder(WeightedAllShortestPaths.ShortestPathTask.class)
59+
.perNode("distance array", Estimate::sizeOfDoubleArray)
60+
.add("priority queue", IntPriorityQueue.memoryEstimation())
61+
.build();
62+
}
63+
}

algo/src/main/java/org/neo4j/gds/allshortestpaths/IntPriorityQueue.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121
import org.jetbrains.annotations.TestOnly;
2222
import org.neo4j.gds.collections.ArrayUtil;
2323
import org.neo4j.gds.collections.ha.HugeIntArray;
24+
import org.neo4j.gds.mem.MemoryEstimation;
25+
import org.neo4j.gds.mem.MemoryEstimations;
2426

2527
/**
2628
* A PriorityQueue specialized for ints that maintains a partial ordering of
@@ -44,6 +46,14 @@ public abstract class IntPriorityQueue {
4446
private final IntLongScatterMap mapElementToIndex;
4547
private long size = 0;
4648

49+
public static MemoryEstimation memoryEstimation() {
50+
return MemoryEstimations.builder(IntPriorityQueue.class)
51+
.perNode("heap", HugeIntArray::memoryEstimation)
52+
.add("costs", MemoryEstimations.builder(IntDoubleScatterMap.class).build())
53+
.add("element to index map", MemoryEstimations.builder(IntLongScatterMap.class).build())
54+
.build();
55+
}
56+
4757
/**
4858
* Creates a new queue with the given capacity.
4959
* The queue dynamically grows to hold all elements.

algo/src/main/java/org/neo4j/gds/allshortestpaths/WeightedAllShortestPaths.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ public Stream<AllShortestPathsStreamResult> compute() {
109109
* Dijkstra Task. Takes one element of the counter at a time
110110
* and starts dijkstra on it.
111111
*/
112-
private final class ShortestPathTask implements Runnable {
112+
public final class ShortestPathTask implements Runnable {
113113

114114
private final IntPriorityQueue queue;
115115
private final double[] distance;
Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
* Copyright (c) "Neo4j"
3+
* Neo4j Sweden AB [http://neo4j.com]
4+
*
5+
* This file is part of Neo4j.
6+
*
7+
* Neo4j is free software: you can redistribute it and/or modify
8+
* it under the terms of the GNU General Public License as published by
9+
* the Free Software Foundation, either version 3 of the License, or
10+
* (at your option) any later version.
11+
*
12+
* This program is distributed in the hope that it will be useful,
13+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
14+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15+
* GNU General Public License for more details.
16+
*
17+
* You should have received a copy of the GNU General Public License
18+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
19+
*/
20+
package org.neo4j.gds.allshortestpaths;
21+
22+
import org.junit.jupiter.params.ParameterizedTest;
23+
import org.junit.jupiter.params.provider.CsvSource;
24+
import org.neo4j.gds.core.concurrency.Concurrency;
25+
26+
import static org.neo4j.gds.assertions.MemoryEstimationAssert.assertThat;
27+
28+
class AllShortestPathsMemoryEstimateDefinitionTest {
29+
30+
@ParameterizedTest
31+
@CsvSource({
32+
"10_000, 1, false, 320",
33+
"10_000, 4, false, 584",
34+
"500_000, 4, false, 584",
35+
"10_000_000, 4, false, 584",
36+
"10_000, 1, true, 120_280",
37+
"10_000, 4, true, 480_976",
38+
"500_000, 4, true, 24_000_976",
39+
"10_000_000, 4, true, 480_000_976"
40+
})
41+
void testMemoryEstimation(long nodeCount, int concurrency, boolean weighted, long expectedMemory) {
42+
var memoryEstimation = new AllShortestPathsMemoryEstimateDefinition(weighted).memoryEstimation();
43+
assertThat(memoryEstimation)
44+
.memoryRange(nodeCount, new Concurrency(concurrency))
45+
.hasSameMinAndMaxEqualTo(expectedMemory);
46+
}
47+
}

applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/PathFindingAlgorithmsEstimationModeBusinessFacade.java

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,8 @@
4949
import org.neo4j.gds.traversal.RandomWalkCountingVisitsMemoryEstimateDefinition;
5050
import org.neo4j.gds.traversal.RandomWalkMemoryEstimateDefinition;
5151
import org.neo4j.gds.traversal.RandomWalkMutateConfig;
52+
import org.neo4j.gds.allshortestpaths.AllShortestPathsMemoryEstimateDefinition;
53+
import org.neo4j.gds.allshortestpaths.AllShortestPathsConfig;
5254

5355
/**
5456
* Here is the top level business facade for all your path finding memory estimation needs.
@@ -61,8 +63,14 @@ public PathFindingAlgorithmsEstimationModeBusinessFacade(AlgorithmEstimationTemp
6163
this.algorithmEstimationTemplate = algorithmEstimationTemplate;
6264
}
6365

64-
MemoryEstimation allShortestPaths() {
65-
throw new MemoryEstimationNotImplementedException();
66+
public MemoryEstimation allShortestPaths(AllShortestPathsConfig configuration) {
67+
return new AllShortestPathsMemoryEstimateDefinition(configuration.hasRelationshipWeightProperty()).memoryEstimation();
68+
}
69+
70+
public MemoryEstimateResult allShortestPaths(AllShortestPathsConfig configuration, Object graphNameOrConfiguration) {
71+
var memoryEstimation = allShortestPaths(configuration);
72+
73+
return runEstimation(configuration, graphNameOrConfiguration, memoryEstimation);
6674
}
6775

6876
public MemoryEstimateResult bellmanFord(AllShortestPathsBellmanFordBaseConfig configuration, Object graphNameOrConfiguration) {

applications/algorithms/path-finding/src/main/java/org/neo4j/gds/applications/algorithms/pathfinding/PathFindingAlgorithmsStreamModeBusinessFacade.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ public <RESULT> Stream<RESULT> allShortestPaths(
9393
graphName,
9494
configuration,
9595
AllShortestPaths,
96-
estimation::allShortestPaths,
96+
() -> estimation.allShortestPaths(configuration),
9797
(graph, __) -> algorithms.allShortestPaths(graph, configuration),
9898
resultBuilder
9999
);

doc/modules/ROOT/pages/algorithms/all-pairs-shortest-path.adoc

Lines changed: 33 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,4 @@
1-
:page-aliases: alpha-algorithms/all-pairs-shortest-path.adoc
2-
31
[[algorithm-all-pairs-shortest-path]]
4-
[.alpha]
52
= All Pairs Shortest Path
63
:description: This section describes the All Pairs Shortest Path algorithm in the Neo4j Graph Data Science library.
74

@@ -10,9 +7,6 @@ include::partial$/product-limitations/not-in-serverless.adoc[]
107
The All Pairs Shortest Path (APSP) calculates the shortest (weighted) path between all pairs of nodes.
118
This algorithm has optimizations that make it quicker than calling the Single Source Shortest Path algorithm for every pair of nodes in the graph.
129

13-
include::partial$/operations-reference/alpha-note.adoc[]
14-
15-
1610
:directed:
1711
:undirected:
1812
:weighted:
@@ -123,6 +117,39 @@ RETURN gds.graph.project(
123117
)
124118
----
125119

120+
[[algorithm-all-pairs-shortest-path-examples-memory-estimation]]
121+
=== Memory Estimation
122+
123+
:mode: stream
124+
include::partial$/algorithms/shared/examples-estimate-intro.adoc[]
125+
126+
[role=query-example]
127+
--
128+
.The following will estimate the memory requirements for running the algorithm:
129+
[source, cypher, role=noplay]
130+
----
131+
CALL gds.allShortestPaths.stream.estimate('cypherGraph', {
132+
relationshipWeightProperty: 'cost'
133+
})
134+
YIELD nodeCount, relationshipCount, bytesMin, bytesMax, requiredMemory
135+
RETURN nodeCount, relationshipCount, bytesMin, bytesMax, requiredMemory
136+
----
137+
138+
.Results
139+
[opts="header"]
140+
|===
141+
| nodeCount | relationshipCount | bytesMin | bytesMax | requiredMemory
142+
| 6 | 18 | 1264 | 1264 | "1264 Bytes"
143+
|===
144+
--
145+
146+
[[algorithm-all-pairs-shortest-path-examples-stream]]
147+
=== Stream
148+
149+
:!stream-details:
150+
In the `stream` execution mode, the algorithm returns the shortest path distance for each source-target pair.
151+
This allows us to inspect the results directly or post-process them in Cypher without any side effects.
152+
126153
[role=query-example]
127154
--
128155
.The following will run the algorithm, treating the graph as undirected:

doc/modules/ROOT/pages/operations-reference/algorithm-references.adoc

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@
2727
| `gds.allShortestPaths.dijkstra.write.estimate` label:procedure[Procedure]
2828
| `gds.allShortestPaths.dijkstra.mutate` label:procedure[Procedure]
2929
| `gds.allShortestPaths.dijkstra.mutate.estimate` label:procedure[Procedure]
30+
.2+<.^| xref:algorithms/all-pairs-shortest-path.adoc[All Shortest Paths]
31+
| `gds.allShortestPaths.stream` label:procedure[Procedure]
32+
| `gds.allShortestPaths.stream.estimate` label:procedure[Procedure]
3033
.8+<.^|xref:algorithms/article-rank.adoc[ArticleRank]
3134
| `gds.articleRank.mutate` label:procedure[Procedure]
3235
| `gds.articleRank.mutate.estimate` label:procedure[Procedure]
@@ -405,8 +408,6 @@
405408
|===
406409
|Algorithm name | Operation
407410
| xref:alpha-algorithms/adamic-adar.adoc[Adamic Adar] | `gds.alpha.linkprediction.adamicAdar` label:function[Function]
408-
.1+<.^|xref:algorithms/all-pairs-shortest-path.adoc[All Shortest Paths]
409-
| `gds.allShortestPaths.stream` label:procedure[Procedure]
410411
| xref:alpha-algorithms/common-neighbors.adoc[Common Neighbors] | `gds.alpha.linkprediction.commonNeighbors` label:function[Function]
411412
.8+<.^|xref:algorithms/harmonic-centrality.adoc[Harmonic Centrality]
412413
| `gds.closeness.harmonic.mutate` label:procedure[Procedure]

open-packaging/src/test/java/org/neo4j/gds/OpenGdsProcedureSmokeTest.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ class OpenGdsProcedureSmokeTest extends BaseProcTest {
5353
"gds.graph.sample.cnarw.estimate",
5454

5555
"gds.allShortestPaths.stream",
56+
"gds.allShortestPaths.stream.estimate",
5657

5758
"gds.articulationPoints.mutate",
5859
"gds.articulationPoints.mutate.estimate",
@@ -619,7 +620,7 @@ void countShouldMatch() {
619620
);
620621

621622
// If you find yourself updating this count, please also update the count in SmokeTest.kt
622-
int expectedCount = 459;
623+
int expectedCount = 460;
623624
assertEquals(
624625
expectedCount,
625626
returnedRows,

proc/path-finding/src/main/java/org/neo4j/gds/paths/all/AllShortestPathsStreamProc.java

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
import org.neo4j.gds.allshortestpaths.AllShortestPathsStreamResult;
2323
import org.neo4j.gds.procedures.GraphDataScienceProcedures;
24+
import org.neo4j.gds.applications.algorithms.machinery.MemoryEstimateResult;
2425
import org.neo4j.procedure.Context;
2526
import org.neo4j.procedure.Description;
2627
import org.neo4j.procedure.Internal;
@@ -31,6 +32,7 @@
3132
import java.util.stream.Stream;
3233

3334
import static org.neo4j.gds.paths.all.Constants.ALL_PAIRS_SHORTEST_PATH_DESCRIPTION;
35+
import static org.neo4j.gds.procedures.ProcedureConstants.MEMORY_ESTIMATION_DESCRIPTION;
3436
import static org.neo4j.procedure.Mode.READ;
3537

3638
public class AllShortestPathsStreamProc {
@@ -46,6 +48,15 @@ public Stream<AllShortestPathsStreamResult> stream(
4648
return facade.algorithms().pathFinding().allShortestPathStream(graphName, configuration);
4749
}
4850

51+
@Procedure(name = "gds.allShortestPaths.stream.estimate", mode = READ)
52+
@Description(MEMORY_ESTIMATION_DESCRIPTION)
53+
public Stream<MemoryEstimateResult> estimate(
54+
@Name(value = "graphNameOrConfiguration") Object graphNameOrConfiguration,
55+
@Name(value = "algoConfiguration") Map<String, Object> algoConfiguration
56+
) {
57+
return facade.algorithms().pathFinding().allShortestPathStreamEstimate(graphNameOrConfiguration, algoConfiguration);
58+
}
59+
4960
@Procedure(name = "gds.alpha.allShortestPaths.stream", mode = READ, deprecatedBy = "gds.allShortestPaths.stream")
5061
@Description(ALL_PAIRS_SHORTEST_PATH_DESCRIPTION)
5162
@Internal

0 commit comments

Comments
 (0)