Skip to content

fix(vendors): fix search and pagination #3522

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
GMishx merged 1 commit into from
Jan 19, 2026
Merged

fix(vendors): fix search and pagination #3522

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/build_and_test.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ jobs:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- uses: webiny/action-conventional-commits@8bc41ff4e7d423d56fa4905f6ff79209a78776c7 # v1.3.0


- name: Verify license headers
run: |
chmod +x .github/testForLicenseHeaders.sh
Expand Down
31 changes: 19 additions & 12 deletions backend/common/src/main/java/org/eclipse/sw360/datahandler/db/ComponentDatabaseHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -371,24 +371,31 @@ public void addSelectLogs(Release release, User user) {
}

public Component getComponent(String id, User user) throws SW360Exception {
Component component = componentRepository.get(id);
try {
Component component = componentRepository.get(id);

if (component == null) {
throw fail(404, "Could not fetch component from database! id=" + id);
}
if (component == null) {
log.error("Component not found in database. Component ID: {}", id);
throw fail(404, "Could not fetch component from database! id=" + id);
}

// Convert Ids to release summary
component.setReleases(releaseRepository.makeSummaryWithPermissions(SummaryType.SUMMARY, component.releaseIds, user));
component.unsetReleaseIds();
component.setReleases(releaseRepository.makeSummaryWithPermissions(SummaryType.SUMMARY, component.releaseIds, user));
component.unsetReleaseIds();

setMainLicenses(component);
setMainLicenses(component);

vendorRepository.fillVendor(component);
vendorRepository.fillVendor(component);

// Set permissions
makePermission(component, user).fillPermissions();
makePermission(component, user).fillPermissions();

return component;
return component;
} catch (SW360Exception e) {
log.error("Error fetching component. Component ID: {}, Error: {}", id, e.getMessage(), e);
throw e;
} catch (Exception e) {
log.error("Unexpected error while fetching component. Component ID: {}, Error: {}", id, e.getMessage(), e);
throw new SW360Exception("Failed to fetch component with id: " + id + ". " + e.getMessage());
}
}

public Component getAccessibleComponent(String id, User user) throws SW360Exception {
Expand Down
72 changes: 68 additions & 4 deletions backend/common/src/main/java/org/eclipse/sw360/datahandler/db/VendorRepository.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,20 +11,25 @@

import static com.google.common.base.Strings.isNullOrEmpty;


import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.eclipse.sw360.datahandler.cloudantclient.DatabaseConnectorCloudant;
import org.eclipse.sw360.datahandler.cloudantclient.DatabaseRepositoryCloudantClient;
import org.eclipse.sw360.datahandler.common.SW360Constants;
import org.eclipse.sw360.datahandler.thrift.PaginationData;
import org.eclipse.sw360.datahandler.thrift.components.Component;
import org.eclipse.sw360.datahandler.thrift.components.Release;
import org.eclipse.sw360.datahandler.thrift.projects.Project;
import org.eclipse.sw360.datahandler.thrift.vendors.Vendor;

import java.util.Set;
import com.ibm.cloud.cloudant.v1.model.DesignDocumentViewsMapReduce;
import org.eclipse.sw360.datahandler.thrift.vendors.VendorSortColumn;
import org.jetbrains.annotations.NotNull;

/**
* CRUD access for the Vendor class
Expand All @@ -47,6 +52,7 @@ public class VendorRepository extends DatabaseRepositoryCloudantClient<Vendor> {
"}";

private static final String ALL = "function(doc) { if (doc.type == 'vendor') emit(null, doc._id) }";
private static final String VENDORS_BY_ALL_IDX = "VendorsByAllIdx";

public VendorRepository(DatabaseConnectorCloudant db) {
super(db, Vendor.class);
Expand All @@ -55,11 +61,20 @@ public VendorRepository(DatabaseConnectorCloudant db) {
views.put("vendorbyshortname", createMapReduce(BY_LOWERCASE_VENDOR_SHORTNAME_VIEW, null));
views.put("vendorbyfullname", createMapReduce(BY_LOWERCASE_VENDOR_FULLNAME_VIEW, null));
initStandardDesignDocument(views, db);

createPartialTypeIndex(
VENDORS_BY_ALL_IDX, "vendorsByType", SW360Constants.TYPE_VENDOR,
new String[]{
Vendor._Fields.TYPE.getFieldName(),
Vendor._Fields.FULLNAME.getFieldName(),
Vendor._Fields.SHORTNAME.getFieldName(),
Vendor._Fields.URL.getFieldName(),
}, db
);
}

public List<Vendor> searchByFullname(String fullname) {
List<Vendor> vendorsMatchingFullname = new ArrayList<Vendor>(get(queryForIdsAsValue("vendorbyfullname", fullname)));
return vendorsMatchingFullname;
return new ArrayList<>(get(queryForIdsAsValue("vendorbyfullname", fullname)));
}

public void fillVendor(Component component) {
Expand All @@ -84,7 +99,7 @@ public void fillVendor(Project project) {
project.unsetVendorId();
}
}

public void fillVendor(Release release) {
fillVendor(release, null);
}
Expand All @@ -109,4 +124,53 @@ public Set<String> getVendorByLowercaseShortnamePrefix(String shortnamePrefix) {
public Set<String> getVendorByLowercaseFullnamePrefix(String fullnamePrefix) {
return queryForIdsByPrefix("vendorbyfullname", fullnamePrefix != null ? fullnamePrefix.toLowerCase() : fullnamePrefix);
}

public Map<PaginationData, List<Vendor>> searchVendorsWithPagination(String searchText, PaginationData pageData) {
if (pageData == null) {
throw new IllegalArgumentException("PaginationData cannot be null");
}

String viewName = getViewFromPagination(pageData);
List<Vendor> vendors;
if (searchText == null || searchText.isBlank()) {
vendors = queryViewPaginated(viewName, pageData, false);
} else {
String prefix = searchText.toLowerCase();
vendors = queryByPrefixPaginated(viewName, prefix, pageData, false);
}

return Collections.singletonMap(pageData, vendors);
}

public Map<PaginationData, List<Vendor>> getVendorsWithPagination(PaginationData pageData) {
if (pageData == null) {
throw new IllegalArgumentException("PaginationData cannot be null");
}

String viewName = getViewFromPagination(pageData);
log.debug("Using view: {} for pagination sort column {}", viewName , pageData.sortColumnNumber);
List<Vendor> vendors = queryViewPaginated(viewName, pageData, false);

return Collections.singletonMap(pageData, vendors);
}


private static @NotNull String getViewFromPagination(PaginationData pageData) {
return switch (VendorSortColumn.findByValue(pageData.getSortColumnNumber())) {
case VendorSortColumn.BY_FULLNAME -> "vendorbyfullname";
case VendorSortColumn.BY_SHORTNAME -> "vendorbyshortname";
case null -> "all";
};
}

private static @NotNull Map<String, String> getSortSelector(PaginationData pageData, boolean ascending) {
return switch (VendorSortColumn.findByValue(pageData.getSortColumnNumber())) {
case VendorSortColumn.BY_FULLNAME -> Collections.singletonMap("fullname", ascending ? "asc" : "desc");
case VendorSortColumn.BY_SHORTNAME -> Collections.singletonMap("shortname", ascending ? "asc" : "desc");
case null, default -> Collections.singletonMap("fullname", ascending ? "asc" : "desc");
};
}



}
100 changes: 84 additions & 16 deletions backend/common/src/main/java/org/eclipse/sw360/datahandler/db/VendorSearchHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,17 +9,25 @@
*/
package org.eclipse.sw360.datahandler.db;

import com.ibm.cloud.cloudant.v1.Cloudant;
import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.collect.FluentIterable;
import com.google.gson.Gson;
import com.ibm.cloud.cloudant.v1.Cloudant;
import org.eclipse.sw360.datahandler.cloudantclient.DatabaseConnectorCloudant;
import org.eclipse.sw360.datahandler.couchdb.lucene.NouveauLuceneAwareDatabaseConnector;
import org.eclipse.sw360.datahandler.thrift.PaginationData;
import org.eclipse.sw360.datahandler.thrift.vendors.Vendor;
import org.eclipse.sw360.datahandler.thrift.vendors.VendorSortColumn;
import org.eclipse.sw360.nouveau.designdocument.NouveauDesignDocument;
import org.eclipse.sw360.nouveau.designdocument.NouveauIndexDesignDocument;
import org.eclipse.sw360.nouveau.designdocument.NouveauIndexFunction;

import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Map;

import static org.eclipse.sw360.datahandler.couchdb.lucene.NouveauLuceneAwareDatabaseConnector.prepareWildcardQuery;
import static org.eclipse.sw360.nouveau.LuceneAwareCouchDbConnector.DEFAULT_DESIGN_PREFIX;
Expand All @@ -36,24 +44,32 @@ public class VendorSearchHandler {
private static final String DDOC_NAME = DEFAULT_DESIGN_PREFIX + "lucene";

private static final NouveauIndexDesignDocument luceneSearchView
= new NouveauIndexDesignDocument("vendors",
= new NouveauIndexDesignDocument("vendors",
new NouveauIndexFunction(
"function(doc) {" +
" if(doc.type == 'vendor') {" +
" if (typeof(doc.shortname) == 'string' && doc.shortname.length > 0) {" +
" index('text', 'shortname', doc.shortname, {'store': true});" +
" }" +
" if (typeof(doc.fullname) == 'string' && doc.fullname.length > 0) {" +
" index('text', 'fullname', doc.fullname, {'store': true});" +
" }" +
" }" +
"}"));
"""
function(doc) {
if(doc.type == 'vendor') {
if (typeof(doc.shortname) == 'string' && doc.shortname.length > 0) {
index('text', 'shortname', doc.shortname, {'store': true});
index('string', 'shortname_sort', doc.shortname);
}
if (typeof(doc.fullname) == 'string' && doc.fullname.length > 0) {
index('text', 'fullname', doc.fullname, {'store': true});
index('string', 'fullname_sort', doc.fullname);
}
}
}
"""
));

private final NouveauLuceneAwareDatabaseConnector connector;
private final VendorRepository vendorRepository;

public VendorSearchHandler(Cloudant client, String dbName) throws IOException {
// Creates the database connector and adds the lucene search view
DatabaseConnectorCloudant db = new DatabaseConnectorCloudant(client, dbName);
// Initialize repository so we have a fallback using the same database and views
vendorRepository = new VendorRepository(db);
connector = new NouveauLuceneAwareDatabaseConnector(db, DDOC_NAME, dbName, db.getInstance().getGson());
Gson gson = db.getInstance().getGson();
NouveauDesignDocument searchView = new NouveauDesignDocument();
Expand All @@ -62,15 +78,67 @@ public VendorSearchHandler(Cloudant client, String dbName) throws IOException {
connector.addDesignDoc(searchView);
}

public List<Vendor> search(String searchText) {
/**
* Get the query with index names for searching vendors. Current indexes available are 'shortname' and 'fullname'.
*
* @param searchText Search query from user.
* @return Lucene search query with index names.
*/
private static @Nonnull String getQueryString(String searchText) {
final Function<String, String> addField = input -> input + ": (" +
prepareWildcardQuery(searchText) + " )";
List<String> typeField = List.of("fullname", "shortname");
return "( " + Joiner.on(" OR ").join(
FluentIterable.from(typeField).transform(addField)
) + " ) ";
}

public Map<PaginationData, List<Vendor>> search(String searchText, PaginationData pageData) {
// Query the search view for the provided text
return connector.searchView(Vendor.class, luceneSearchView.getIndexName(),
prepareWildcardQuery(searchText));
String sortColumn = getSortColumnName(pageData);
Map<PaginationData, List<Vendor>> luceneResult = connector.searchView(
Vendor.class,
luceneSearchView.getIndexName(),
getQueryString(searchText),
pageData,
sortColumn,
pageData.isAscending());

if (hasResults(luceneResult)) {
return luceneResult;
}

// Fallback to simple in-memory filtering when lucene is unavailable (e.g. in tests)
return vendorRepository.searchVendorsWithPagination(searchText, pageData);
}

public List<String> searchIds(String searchText) {
// Query the search view for the provided text
return connector.searchIds(Vendor.class, luceneSearchView.getIndexName(),
prepareWildcardQuery(searchText));
getQueryString(searchText));
}



private static boolean hasResults(Map<PaginationData, List<Vendor>> luceneResult) {
return luceneResult != null
&& !luceneResult.isEmpty()
&& luceneResult.values().stream().anyMatch(list -> list != null && !list.isEmpty());
}


/**
* Convert sort column number back to sorting column name. This function makes sure to use the string column (with
* `_sort` suffix) for text indexes.
* @param pageData Pagination Data from the request.
* @return Sort column name. Defaults to fullname_sort
*/
private static @Nonnull String getSortColumnName(@Nonnull PaginationData pageData) {
return switch (VendorSortColumn.findByValue(pageData.getSortColumnNumber())) {
case VendorSortColumn.BY_SHORTNAME -> "shortname_sort";
case VendorSortColumn.BY_FULLNAME -> "fullname_sort";
case null -> "fullname_sort";
default -> "fullname_sort";
};
}
}
16 changes: 11 additions & 5 deletions backend/vendors/src/main/java/org/eclipse/sw360/vendors/VendorDatabaseHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import org.eclipse.sw360.datahandler.db.VendorRepository;
import org.eclipse.sw360.datahandler.thrift.AddDocumentRequestStatus;
import org.eclipse.sw360.datahandler.thrift.AddDocumentRequestSummary;
import org.eclipse.sw360.datahandler.thrift.PaginationData;
import org.eclipse.sw360.datahandler.thrift.RequestStatus;
import org.eclipse.sw360.datahandler.thrift.RequestSummary;
import org.eclipse.sw360.datahandler.thrift.SW360Exception;
Expand All @@ -39,6 +40,7 @@
import java.nio.ByteBuffer;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import com.ibm.cloud.cloudant.v1.Cloudant;
Expand Down Expand Up @@ -69,6 +71,10 @@ public List<Vendor> getAllVendors() throws TException {
return repository.getAll();
}

public Map<PaginationData, List<Vendor>> getAllVendors(PaginationData pageData) throws TException {
return repository.getVendorsWithPagination(pageData);
}

public AddDocumentRequestSummary addVendor(Vendor vendor) {
trimVendorFields(vendor);
if (isDuplicate(vendor)) {
Expand Down Expand Up @@ -152,7 +158,7 @@ public void fillVendor(Release release){
public RequestStatus mergeVendors(String mergeTargetId, String mergeSourceId, Vendor mergeSelection, User user) throws TException {
Vendor mergeTarget = getByID(mergeTargetId);
Vendor mergeSource = getByID(mergeSourceId);

if (!makePermission(mergeTarget, user).isActionAllowed(RequestedAction.WRITE)
|| !makePermission(mergeSource, user).isActionAllowed(RequestedAction.WRITE)
|| !makePermission(mergeSource, user).isActionAllowed(RequestedAction.DELETE)) {
Expand All @@ -173,18 +179,18 @@ public RequestStatus mergeVendors(String mergeTargetId, String mergeSourceId, Ve
if(status != RequestStatus.SUCCESS) {
return status;
}

// now update the vendor in relating documents
summary = updateComponents(mergeTarget, mergeSource, user);
if(summary.getRequestStatus() != RequestStatus.SUCCESS) {
log.error("Cannot update [" + (summary.getTotalElements() - summary.getTotalAffectedElements())
log.error("Cannot update [" + (summary.getTotalElements() - summary.getTotalAffectedElements())
+ "] of [" + summary.getTotalElements() + "] components: " + summary.getMessage());
return summary.getRequestStatus();
}

summary = updateReleases(mergeTarget, mergeSource, user);
if(summary.getRequestStatus() != RequestStatus.SUCCESS) {
log.error("Cannot update [" + (summary.getTotalElements() - summary.getTotalAffectedElements())
log.error("Cannot update [" + (summary.getTotalElements() - summary.getTotalAffectedElements())
+ "] of [" + summary.getTotalElements() + "] releases: " + summary.getMessage());
return summary.getRequestStatus();
}
Expand Down Expand Up @@ -213,7 +219,7 @@ private RequestSummary updateComponents(Vendor mergeTarget, Vendor mergeSource,
components.stream().forEach(component -> {
component.setDefaultVendorId(mergeTarget.getId());
});

return componentsClient.updateComponents(components, user);
}

Expand Down
Loading