Merge branch 'osmandapp:master' into hardy_Afa+searchprio

Author: sonora

OsmAnd-java/src/main/java/net/osmand/search/core/spatial/SpatialPoiSearch.java

@@ -3,7 +3,9 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Iterator;
 import java.util.LinkedHashMap;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
@@ -66,6 +68,21 @@ public SpatialPoiType(String additional, String key, int id) {
 			this.id = id;
 			this.poiAdditional = additional;
 		}
+
+		public boolean accept(Amenity a) {
+			if (key.equals(a.getType().getKeyName())) {
+				return true;
+			}
+			// contains for ';'
+			boolean multi = a.getSubType().indexOf(';') != -1;
+			if (key.equals(a.getSubType()) 
+					|| (multi && key.startsWith(a.getSubType() + ";"))
+					|| (multi && key.endsWith(";" + a.getSubType()))
+					|| (multi && key.contains(";" + a.getSubType() + ";"))) {
+				return true;
+			}
+			return false;
+		}
 
 	}
 
@@ -288,6 +305,21 @@ public SpatialPoiType getByKey(String key) {
 	}
 
 
+	private List<BinaryMapIndexReader> filterByRadius(LatLon l, int rad, List<BinaryMapIndexReader> oFiles,
+			List<BinaryMapIndexReader> res) {
+		QuadRect rect = MapUtils.calculateBbox(rad, l);
+		Iterator<BinaryMapIndexReader> it = oFiles.iterator();
+		while (it.hasNext()) {
+			BinaryMapIndexReader next = it.next();
+			if (next.containsPoiData((int) rect.left, (int) rect.top, (int) rect.right, (int) rect.bottom)) {
+				res.add(next);
+				it.remove();
+			}
+		}
+		return res;
+	}
+
+
 	public List<Amenity> loadPOIObjects(SpatialSearchContext ctx, long id, LatLon latLon, int radMeters, int limit)
 			throws IOException {
 		final SpatialPoiType spt = byId.get((int) id);
@@ -356,7 +388,8 @@ public boolean publish(Amenity object) {
 
 				@Override
 				public boolean isCancelled() {
-					return alimit[0] == 0;
+					return false; // allow to read all for proper sorting
+//					return alimit[0] == 0;
 				}
 			};
 			SearchRequest<Amenity> req = BinaryMapIndexReader.buildSearchPoiRequest(
@@ -366,16 +399,29 @@ public boolean isCancelled() {
 				req = BinaryMapIndexReader.buildSearchPoiRequest((int) qr.left, (int) qr.right, (int) qr.top,
 						(int) qr.bottom, -1, typeFilter, addFilter, matcher);
 			}
-			for (BinaryMapIndexReader bir : ctx.files) {
-				ctx.stats.poiByTypeTime.start();
-				long br = bir.getBytesRead();
-				bir.searchPoi(req);
-				ctx.stats.poiByTypeBytes += (bir.getBytesRead() - br);
-				ctx.stats.poiByTypeTime.finish();
-				ctx.stats.poiByTypeBboxes += req.numberOfReadSubtrees;
+			List<BinaryMapIndexReader> oFiles = new LinkedList<BinaryMapIndexReader>(ctx.files);
+			iterateSearch(ctx, req, filterByRadius(latLon, 5_000, oFiles, new ArrayList<BinaryMapIndexReader>()));
+			if (alimit[0] != 0 && radMeters > 5_000) {
+				iterateSearch(ctx, req, filterByRadius(latLon, 50_000, oFiles, new ArrayList<BinaryMapIndexReader>()));
+			}
+			if (alimit[0] != 0 && radMeters > 50_000) {
+				iterateSearch(ctx, req, oFiles);
 			}
 		}
 		return results;
 	}
 
+
+	private void iterateSearch(SpatialSearchContext ctx, SearchRequest<Amenity> req, List<BinaryMapIndexReader> res)
+			throws IOException {
+		for (BinaryMapIndexReader bir : res) {
+			ctx.stats.poiByTypeTime.start();
+			long br = bir.getBytesRead();
+			bir.searchPoi(req);
+			ctx.stats.poiByTypeBytes += (bir.getBytesRead() - br);
+			ctx.stats.poiByTypeTime.finish();
+			ctx.stats.poiByTypeBboxes += req.numberOfReadSubtrees;
+		}
+	}
+
 }

OsmAnd-java/src/main/java/net/osmand/search/core/spatial/SpatialSearchResultsList.java

@@ -25,6 +25,7 @@
 import net.osmand.data.MapObject;
 import net.osmand.data.Street;
 import net.osmand.search.core.HashQuadTree;
+import net.osmand.search.core.spatial.SpatialPoiSearch.SpatialPoiType;
 import net.osmand.search.core.spatial.SpatialSearchToken.NameIndexAtom;
 import net.osmand.search.core.spatial.SpatialTextSearch.SpatialTextSearchSettings;
 import net.osmand.util.MapUtils;
@@ -145,6 +146,16 @@ public List<SpatialSearchToken> getMissingTokens(SpatialSearchContext ctx) {
 	public void loadObjectsAndCalcBuildings(SpatialSearchContext ctx) throws IOException {
 		ctx.stats.sub2LoadObjectsBldTime.start();
 		loadObjects(ctx);
+		// calculate amenity type
+		for (int indx = 0; indx < getCombinations(); indx++) {
+			if (skipResults.contains(indx)) {
+				continue;
+			}
+			boolean skip = checkAmenityType(ctx, indx);
+			if (!skip) {
+				skipResults.put(indx, true);
+			}
+		}
 
 		if (ctx.settings.SEARCH_BUILDINGS) {
 			Map<String, Building> bldCheckCache = new HashMap<>();
@@ -167,6 +178,50 @@ public void loadObjectsAndCalcBuildings(SpatialSearchContext ctx) throws IOExcep
 
 		ctx.stats.sub2LoadObjectsBldTime.finish();
 	}
+
+	private boolean checkAmenityType(SpatialSearchContext ctx, int indx) {
+		SpatialPoiType type = null;
+		for (int i = 0; i < tCount; i++) {
+			NameIndexAtom atom = linearResults.get(indx * tCount + i);
+			if (atom.isPoiCategory()) {
+				type = ctx.poiSearch.getById((int) atom.id);
+				break;
+			}
+		}
+		if (type == null) {
+			return true;
+		}
+		for (int i = 0; i < tCount; i++) {
+			NameIndexAtom atom = linearResults.get(indx * tCount + i);
+			if (atom.object instanceof Amenity a) {
+				boolean matched = type.accept(a);
+				if (!matched) {
+					return false;
+				}
+				// old logic for missing tokens
+//				for (SpatialSearchToken st : missingTokens) {
+//					if (st.hasPoiType(a.getType().getKeyName(), ctx.poiSearch) != null
+//							|| st.hasPoiType(a.getSubType(), ctx.poiSearch) != null) {
+//						matched = true;
+//						break;
+//					}
+//					if (a.getSubType().indexOf(';') != -1) {
+//						for (String subtStr : a.getSubType().split(";")) {
+//							if (st.hasPoiType(subtStr, ctx.poiSearch) != null) {
+//								matched = true;
+//								break;
+//							}
+//						}
+//					}
+//				}
+//				if (matched) {
+//					atom.otherWordsCnt = -1; // could be added to surplusWords later 
+//					break;
+//				}
+			}
+		}
+		return true;
+	}
 
 	private void calcStreetIntersections(SpatialSearchContext ctx, int indx) {
 		NameIndexAtom first = null;
@@ -673,13 +728,15 @@ private boolean acceptIntersection(SpatialSearchContext ctx,  SpatialSearchResul
 			} else if ((a.buildingInd >= 0) && pa.isStreetBuilding() && !a.isCityStreetName()) {
 				return false; 
 			}
-			// if poi doesn't have bbox don't intersect or add bbox!
-			if ((pa.buildingInd >= 0) && a.isPOI() && a.coords.bbox31 == null) {
-				return false;
-			} else if ((a.buildingInd >= 0) && pa.isPOI() && pa.coords.bbox31 == null) {
-				return false;
+			// if poi doesn't have bbox don't intersect or add bbox! (transport stops take street names)
+			if (!ctx.settings.TEST_ALLOW_HOUSE_POI_TYPE_INTERSECTION) {
+				if ((pa.buildingInd >= 0) && a.isPOI() && a.coords.bbox31 == null) {
+					return false;
+				} else if ((a.buildingInd >= 0) && pa.isPOI() && pa.coords.bbox31 == null) {
+					return false;
+				}
 			}
-			
+//			
 		}
 		// 2. Duplicate words in query
 		for (int i = 0; parent != null && i < parent.tCount; i++) {
@@ -721,6 +778,7 @@ private boolean acceptIntersection(SpatialSearchContext ctx,  SpatialSearchResul
 			atomObjs.put(a.id, a);
 		}
 		NameIndexAtom poiType = a.isPoiCategory() ? a : null;
+		SpatialSearchToken poiTypeToken = tokens[0];
 		for (int i = 0; parent != null && i < parent.tCount; i++) {
 			NameIndexAtom pa = parent.linearResults.get(pindx * parent.tCount + i);
 			if (pa.id == a.id) {
@@ -729,6 +787,7 @@ private boolean acceptIntersection(SpatialSearchContext ctx,  SpatialSearchResul
 			if (pa.isPoiCategory()) {
 				if (poiType == null || pa.id == a.id) {
 					poiType = pa;
+					poiTypeToken = parent.tokens[i];
 				} else {
 					// no 2 poi categories
 					return false;
@@ -758,8 +817,15 @@ private boolean acceptIntersection(SpatialSearchContext ctx,  SpatialSearchResul
 //				return false;
 //			}
 		}
+		// investigate to find types
 		if (poiType != null && atomObjs.size() > 0) {
-			return false;
+			NameIndexAtom p = atomObjs.values().iterator().next();
+			if (poiTypeToken.incomplete) {
+				return false;
+			}
+			if (!p.isPOI()) {
+				return false;
+			}
 		}
 		if (atomObjs.size() > 1) {
 			Iterator<NameIndexAtom> it = atomObjs.values().iterator();

OsmAnd-java/src/main/java/net/osmand/search/core/spatial/SpatialSearchTestAndDocs.java

@@ -37,28 +37,24 @@
 // TESTING delete default enlarge and enlarge data
 // TESTING Venezia city Street / Place  -  <City Street> ('<Salt Lake City>') with Street ('Pennsylvania street') 
 // TESTING find check that token is reused in parent - and ignore intersection for complete mattch
-// TESTING TODO WEB ! POI Categories + top poi categories !! RZR
 // TESTING POI CATEGORY Specific Healthcare specialties (Vegan) - https://github.com/osmandapp/OsmAnd/issues/24941
 // TESTING BUG: numbers obj- filter cafe & rest  + incorrect PrivatBank counts
-// TESTING: check additional filter not stored old
+// TESTING check additional filter not stored old
+// TESTING Sort maps poi categories API search (sort bboxes?)
+// TESTING query = "Church Catedral-Basílica de Nuestra Señora del Pilar"; -  POI_TYPE /\ POI (SYNONYMS!)
+// TEST_ALLOW_HOUSE_POI_TYPE_INTERSECTION Review if poi doesn't have bbox don't intersect or add bbox! - Shell 2 Rožňavská (test)
 
 ////////// IN PROGRESS //////////
 
-// TODO POI: Sort maps poi categories API search (sort bboxes?)
-
-// TODO POI Categories translations / synonyms (WEB) - Стоматол., Dentist, Stomatology 
-// TODO query = "Catedral-Basílica de Nuestra Señora del Pilar"; -  POI_TYPE /\ POI
-// TODO Review if poi doesn't have bbox don't intersect or add bbox! - Shell 2 Rožňavská
-
-// TODO INSPECTOR stats index_words_dashboard.html
+// INSPECTOR stats index_words_dashboard.html
 
 // TO DO Ivan / Gateway
 // TODO DEDUPLICATE: Review / implement similarity radius - similarityRadius = 50000 ... Route Id
 // TODO DEDUPLICATE: Unite RouteArticle, POI by wikidata id ? - DEPTH_TO_CHECK_SAME_SEARCH_RESULTS = 20;...
 // TODO DEDUPLICATE: Venezia ? - No place=city in POI is it on purpose ? 2 Wikidataids! Rating not merged. POI - relation/44741 (Q641), CITY - way/64778090 (Q33723961).
 // TODO DEDUPLICATE: review osm route id  combine by?
 // TODO DEDUPLICATE: Index place=state, county.. + wikidata id for boundaries (regions.ocbf) & display them - analyze
-// TODO DEDUPLICATE: Test wiki / travel maps, seamarks map
+// TODO DEDUPLICATE: Test wiki / travel maps / seamarks map
 // TODO DEDUPLICATE: same location (5-10m) 2 streets different cities
 // TODO DEDUPLICATE: brand langs - 'Поїхали з нами' / 'Поехали с нами'
 // TODO UNIT TESTS: (duplicate words), Бульварно-Кудрявська, NC-42, 2-га Нова (2 Нова), M2...
@@ -68,21 +64,24 @@
 // TODO INSPECTOR : doesn't show suffixes
 
 // LARGE IMPORTANT TASKS
-// TODO ANALYZE: find slow queries on Autotests
+// TODO INDEX: Speedup load after sorting - to limit objects (store elo in index)! 
+// TODO INDEX: Store Poi category index (effective intersection 'Church St. Miguel' - refactor checkAmenity)
+// TODO INDEX: Find POI Categories translations / synonyms (WEB) - Стоматол., Dentist, Stomatology, Basilica (?)
+// TODO ANALYZE: BUG - Germany POI words - . (115,158, 115,158), und (97,839, 97,839), - not common? - bach (56,475, 56,475) - could be common?
 // TODO ANALYZE: too many wiki places on streets?
-// TODO ANALYZE: Germany POI words - . (115,158, 115,158), und (97,839, 97,839), - not common? - bach (56,475, 56,475) - could be common?
-// TODO POI CATEGORY Bboxes too large - test size OsmAndPoiNameIndexDataAtom, quad tree (90% < 10K) add rare categories 
-// TODO ANALYZE: Large Geo atoms "Berlin" 
+// TODO ANALYZE: find slow queries on Autotests
+// TODO ANALYZE: Large Geo atoms "Berlin" (Slow query)
 
 // TO DO - RZR
-// TODO WEB PRODUCTION: display results std way: house, interpolation results, poi...
-// TODO WEB Production: Multithread pool, Monitor / time & memory optimize memory?
+// TODO WEB: POI Categories + top poi categories
+// TODO WEB: display results std way: house, interpolation results, poi...
+// TODO WEB: Multithread pool, Monitor / time & memory optimize memory?
 // TODO ANDROID: Integrate (include regions.ocbf) on client
 // TODO ANDROID: Progress / cancel
 // TODO ANDROID: memory performance 
-// 
-
+ 
 /////////////// EXTRA FEATURES ///////////////
+// TODO OBF POI CATEGORY Bboxes too large - investigate size (introduce for categories OBF) - OsmAndPoiNameIndexDataAtom, quad tree (90% < 10K)
 // TODO Review Abbrevations (synonyms / direction words) other languages?
 // TODO Store and test conscription number for some cities - issue (RZR)
 // TODO Search in large parks, neighborhood same as in boundaries (index bbox POI), residential way/56238205
@@ -168,6 +167,7 @@ public static void main(String[] args) throws IOException, InterruptedException
 //		pattern = "Map";
 		String pattern2 = ".....";
 		String query = "Berlin hauptstrasse"; // slow
+		query = "Berlin";
 //		query = "Kelterstraße Kernen im Remstal";
 //		query = "Germany Kelter. Kernen im Remstal";
 //		query = "3 Hofäckerstraße Kernen im Remstal";
@@ -272,19 +272,18 @@ public static void main(String[] args) throws IOException, InterruptedException
 
 //		pattern = "regions.ocbf" ;
 
-		pattern = "Ukraine_kyiv-city";
+//		pattern = "Ukraine_kyiv-city";
 //		pattern = "Test_Ukraine_kyiv-city_europe_12.obf";
-//		pattern = "Ukraine_";
+		pattern = "Ukraine_";
 		// poi types
 //		location = new LatLon(50.439, 30.516);
-		settings.SEARCH_POI = false;
-		settings.DEV_PRINT_POI_CAT_LIMIT = 100000; 
-		settings.DEV_PRINT_POI_CAT_RADIUS_KM = 1000;
-		query = "п.";
-		query = "New york.";
+//		settings.SEARCH_POI = false;
+//		settings.DEV_PRINT_POI_CAT_LIMIT = 1000; 
+//		settings.DEV_PRINT_POI_CAT_RADIUS_KM = 10;
+//		query = "при.";
 //		query = "Cafe";
 //		query = "Aquarium.";
-//		query = "Vegeterian";
+//		query = "Veget.";
 //		query = "Mcdonald's";
 //		query = "Stomat.";
 
@@ -332,6 +331,7 @@ public static void main(String[] args) throws IOException, InterruptedException
 //		pattern = "Slovakia";
 //		query = "Bratislava Billa";
 //		settings.DEDUPLICATE_RES = false;
+//		settings.TEST_ALLOW_HOUSE_POI_TYPE_INTERSECTION = false;
 //		query = "Shell 2 Rožňavská";
 
 //		pattern = "Us_new-york_new"; // new-york, new-jersey
@@ -389,9 +389,10 @@ public static void main(String[] args) throws IOException, InterruptedException
 //		query = "Венец."; 
 
 //		pattern = "Spain_aragon_europe_";
-//		query = "Basílica de Nuestra Señora del Pilar";
+//		query = "Church Basílica de Nuestra Señora del Pilar"; // Church vs Roman Church
 //		query = "Catedral-Basílica de Nuestra Señora del Pilar"; // 7 words! 2^7 combinations
-		
+//		query = "Square de Nuestra Señora del Pilar";  // Church vs Square
+//		
 //		pattern = "Peru_";
 //		query ="Calle 20 188 San Isidro Lima"; // 1430799557
 //		query ="Lima Calle 20 San Isidro";
@@ -414,7 +415,7 @@ public static void main(String[] args) throws IOException, InterruptedException
 //		settings.DEDUPLICATE_RES = false;
 		SpatialPoiSearch poiSearch = new SpatialPoiSearch(MapPoiTypes.getDefault());
 		SpatialSearchContext searchContext = new SpatialSearchContext(settings, ls, poiSearch, location);
-		SpatialSearchResults rs = a.searchTest(query, searchContext, 10);
+		SpatialSearchResults rs = a.searchTest(query, searchContext, 10000);
 		SpatialSearchResult mainResult = rs.getFirstResult();
 		if (mainResult != null && mainResult.matchedTokens() < rs.tokens.size() - 2) {
 			// another way to check to check to get mainResult - boundary object