Merge pull request #1246 from bcc-code/feat/remove-algolia

Feat/remove algolia

Author: KillerX

CLAUDE.md

@@ -0,0 +1,2 @@
+- The SQL queries always need to be written into the .sql files and the go files generated using sqlc.
+- To test the GQL interface locally, you can use curl to the address http://localhost:8080/query.

backend/cmd/api/env.go

@@ -211,10 +211,6 @@ func getEnvConfig() envConfig {
 		},
 		Port: os.Getenv("PORT"),
 		Search: search.Config{
-			Algolia: search.AlgoliaConfig{
-				AppID:  os.Getenv("ALGOLIA_APP_ID"),
-				APIKey: os.Getenv("ALGOLIA_API_KEY"),
-			},
 			Elastic: search.ElasticConfig{
 				CloudID: os.Getenv("ELASTIC_CLOUDID"),
 				ApiKey:  os.Getenv("ELASTIC_APIKEY"),

backend/cmd/jobs/env.go

@@ -117,10 +117,6 @@ func getEnvConfig() envConfig {
 			MaxIdleConnections: utils.AsIntOrNil(os.Getenv("DB_MAX_IDLE_CONS")),
 		},
 		Search: search.Config{
-			Algolia: search.AlgoliaConfig{
-				AppID:  os.Getenv("ALGOLIA_APP_ID"),
-				APIKey: os.Getenv("ALGOLIA_API_KEY"),
-			},
 			Elastic: search.ElasticConfig{
 				CloudID: os.Getenv("ELASTIC_CLOUDID"),
 				ApiKey:  os.Getenv("ELASTIC_APIKEY"),

backend/cmd/jobs/server/server.go

@@ -142,7 +142,6 @@ func (s Server) ProcessMessage(c *gin.Context) {
 	case events.TypeDirectusEvent:
 		err = s.services.GetEventHandler().ProcessDirectusEvent(ctx, e)
 	case events.TypeSearchReindex:
-		err = s.services.GetSearchService().Reindex(ctx)
 		err = s.services.GetSearchService().ReindexElastic(ctx)
 	case events.TypeExportAnswersToBQ:
 		err = s.services.GetStatisticHandler().HandleAnswerExportToBQ(ctx)

backend/graph/api/resolver.go

@@ -48,7 +48,6 @@ const timestampContextKey = "GqlTimestamp"
 const episodeContextKey = "EpisodeContext"
 
 type searchProvider interface {
-	Search(ctx *gin.Context, query common.SearchQuery, userToken string) (searchResult common.SearchResult, err error)
 	SearchElastic(ctx *gin.Context, query common.SearchQuery, userToken string) (searchResult common.SearchResult, err error)
 }
 

backend/graph/api/search-resolver.go

@@ -5,7 +5,6 @@ import (
 	"github.com/bcc-code/bcc-media-platform/backend/common"
 	"github.com/bcc-code/bcc-media-platform/backend/graph/api/model"
 	"github.com/bcc-code/bcc-media-platform/backend/ratelimit"
-	"github.com/bcc-code/bcc-media-platform/backend/unleash"
 	"github.com/bcc-code/bcc-media-platform/backend/user"
 	"github.com/bcc-code/bcc-media-platform/backend/utils"
 	"strconv"
@@ -161,30 +160,14 @@ func searchResolver(r *queryRootResolver, ctx context.Context, queryString strin
 		return nil, err
 	}
 
-	flags := utils.GetFeatureFlags(ginCtx)
-	var searchResult common.SearchResult
-
-	searchProvider := "unknown"
-	if value, ok := flags.GetVariant(unleash.ElasticSearchFlag); ok && value == unleash.ElasticSearchEnabledVariant {
-		utils.ReportFlagActivation(ginCtx, unleash.ElasticSearchFlag, unleash.ElasticSearchEnabledVariant)
-		searchResult, err = r.SearchService.SearchElastic(ginCtx, common.SearchQuery{
-			Query:    queryString,
-			Limit:    first,
-			Offset:   offset,
-			Type:     typeArg,
-			MinScore: minScore,
-		}, r.AnalyticsIDFactory(ctx))
-		searchProvider = "elastic"
-	} else {
-		searchResult, err = r.SearchService.Search(ginCtx, common.SearchQuery{
-			Query:    queryString,
-			Limit:    first,
-			Offset:   offset,
-			Type:     typeArg,
-			MinScore: minScore,
-		}, r.AnalyticsIDFactory(ctx))
-		searchProvider = "algolia"
-	}
+	searchProvider := "elastic"
+	searchResult, err := r.SearchService.SearchElastic(ginCtx, common.SearchQuery{
+		Query:    queryString,
+		Limit:    first,
+		Offset:   offset,
+		Type:     typeArg,
+		MinScore: minScore,
+	}, r.AnalyticsIDFactory(ctx))
 
 	if err != nil {
 		return nil, err

backend/search/indexing.go

@@ -3,18 +3,14 @@ package search
 import (
 	"bytes"
 	"context"
-	"github.com/algolia/algoliasearch-client-go/v3/algolia/opt"
-	"github.com/algolia/algoliasearch-client-go/v3/algolia/search"
 	"github.com/ansel1/merry/v2"
 	"github.com/bcc-code/bcc-media-platform/backend/common"
 	"github.com/bcc-code/bcc-media-platform/backend/loaders"
 	"github.com/bcc-code/bcc-media-platform/backend/log"
 	"github.com/elastic/go-elasticsearch/v8"
-	"github.com/google/uuid"
 	"github.com/orsinium-labs/enum"
 	"github.com/samber/lo"
 	"strconv"
-	"strings"
 )
 
 const tempIndexName = "bccm-temp"
@@ -111,113 +107,6 @@ func (service *Service) ReindexElastic(ctx context.Context) error {
 	return nil
 }
 
-// Reindex every supported collection with Algolia
-func (service *Service) Reindex(ctx context.Context) error {
-	res, err := service.algoliaClient.CopyIndex(indexName, tempIndexName)
-	if err != nil {
-		return err
-	}
-	_ = res.Wait()
-
-	index := service.algoliaClient.InitIndex(tempIndexName)
-	_, err = index.ClearObjects()
-	if err != nil {
-		return err
-	}
-
-	// Make sure we're not fetching from cache anywhere,
-	// although that shouldn't be an issue, as we're priming on fetch anyway
-	service.loaders.ShowLoader.ClearAll()
-	service.loaders.ShowPermissionLoader.ClearAll()
-	service.loaders.SeasonLoader.ClearAll()
-	service.loaders.SeasonPermissionLoader.ClearAll()
-	service.loaders.EpisodeLoader.ClearAll()
-	service.loaders.EpisodePermissionLoader.ClearAll()
-	service.loaders.PlaylistLoader.ClearAll()
-	service.loaders.PlaylistPermissionLoader.ClearAll()
-
-	// Makes it possible to filter in query, which fields you are searching on
-	// Also configures hits per page
-	titleFields, err := service.getTranslatedTitleFields()
-	if err != nil {
-		return err
-	}
-	descriptionFields, err := service.getTranslatedDescriptionFields()
-	if err != nil {
-		return err
-	}
-	relationalFields, err := service.getRelationalTranslatedFields()
-	if err != nil {
-		return err
-	}
-	searchableAttributes := opt.SearchableAttributes(
-		strings.Join(titleFields, ", "),
-		tagsField,
-		strings.Join(descriptionFields, ", "),
-		strings.Join(relationalFields, ", "),
-		strings.Join(getFunctionalFields(), ", "),
-	)
-	languages, err := service.getLanguageKeys()
-	if err != nil {
-		return err
-	}
-
-	supportedLanguages := []string{"da", "de", "en", "es", "fi", "fr", "hu", "it", "nl", "no", "pl", "pt", "ro", "ru", "tr"}
-
-	languages = lo.Filter(languages, func(l string, _ int) bool {
-		return lo.Contains(supportedLanguages, l)
-	})
-
-	_, err = index.SetSettings(search.Settings{
-		IndexLanguages:        opt.IndexLanguages(languages...),
-		QueryLanguages:        opt.QueryLanguages(languages...),
-		SearchableAttributes:  searchableAttributes,
-		AttributesForFaceting: opt.AttributesForFaceting(service.getFilterFields()...),
-		HitsPerPage:           opt.HitsPerPage(hitsPerPage),
-	})
-	if err != nil {
-		return err
-	}
-
-	log.L.Debug().Str("collection", "shows").Msg("Indexing")
-	err = service.indexShows(ctx, index)
-	if err != nil {
-		return err
-	}
-	log.L.Debug().Str("collection", "seasons").Msg("Indexing")
-	err = service.indexSeasons(ctx, index)
-	if err != nil {
-		return err
-	}
-	log.L.Debug().Str("collection", "episodes").Msg("Indexing")
-	err = service.indexEpisodes(ctx, index)
-	if err != nil {
-		return err
-	}
-	log.L.Debug().Str("collection", "episodes").Msg("Indexing")
-	err = service.indexPlaylists(ctx, index)
-	if err != nil {
-		return err
-	}
-
-	res, err = service.algoliaClient.MoveIndex(tempIndexName, indexName)
-	if err != nil {
-		return err
-	}
-	return res.Wait()
-}
-
-func (service *Service) indexShows(ctx context.Context, index *search.Index) error {
-	return indexCollection[int, common.Show](
-		ctx,
-		index,
-		service.loaders.ShowLoader,
-		service.loaders.ShowPermissionLoader,
-		service.queries.ListShows,
-		service.showToSearchItem,
-	)
-}
-
 func (service *Service) indexShow(ctx context.Context, id int) error {
 	i, err := service.loaders.ShowLoader.Load(ctx, id)()
 	if err != nil {
@@ -228,23 +117,7 @@ func (service *Service) indexShow(ctx context.Context, id int) error {
 		return err
 	}
 
-	err = indexObjectElastic[int, common.Show](ctx, service.elasticClient, IndexShows, *i, p, service.showToSearchItem)
-	if err != nil {
-		log.L.Error().Err(err).Msg("Could not index show in elastic")
-	}
-
-	return indexObject[int, common.Show](ctx, service, *i, p, service.showToSearchItem)
-}
-
-func (service *Service) indexSeasons(ctx context.Context, index *search.Index) error {
-	return indexCollection[int, common.Season](
-		ctx,
-		index,
-		service.loaders.SeasonLoader,
-		service.loaders.SeasonPermissionLoader,
-		service.queries.ListSeasons,
-		service.seasonToSearchItem,
-	)
+	return indexObjectElastic[int, common.Show](ctx, service.elasticClient, IndexShows, *i, p, service.showToSearchItem)
 }
 
 func (service *Service) indexSeason(ctx context.Context, id int) error {
@@ -257,22 +130,7 @@ func (service *Service) indexSeason(ctx context.Context, id int) error {
 		return err
 	}
 
-	err = indexObjectElastic[int, common.Season](ctx, service.elasticClient, IndexSeasons, *i, p, service.seasonToSearchItem)
-	if err != nil {
-		log.L.Error().Err(err).Msg("Could not index season in elastic")
-	}
-	return indexObject[int, common.Season](ctx, service, *i, p, service.seasonToSearchItem)
-}
-
-func (service *Service) indexEpisodes(ctx context.Context, index *search.Index) error {
-	return indexCollection[int, common.Episode](
-		ctx,
-		index,
-		service.loaders.EpisodeLoader,
-		service.loaders.EpisodePermissionLoader,
-		service.queries.ListEpisodes,
-		service.episodeToSearchItem,
-	)
+	return indexObjectElastic[int, common.Season](ctx, service.elasticClient, IndexSeasons, *i, p, service.seasonToSearchItem)
 }
 
 func (service *Service) indexEpisode(ctx context.Context, id int) error {
@@ -285,22 +143,7 @@ func (service *Service) indexEpisode(ctx context.Context, id int) error {
 		return err
 	}
 
-	err = indexObjectElastic[int, common.Episode](ctx, service.elasticClient, IndexEpisodes, *i, p, service.episodeToSearchItem)
-	if err != nil {
-		log.L.Error().Err(err).Msg("Could not index episode in elastic")
-	}
-	return indexObject[int, common.Episode](ctx, service, *i, p, service.episodeToSearchItem)
-}
-
-func (service *Service) indexPlaylists(ctx context.Context, index *search.Index) error {
-	return indexCollection[uuid.UUID, common.Playlist](
-		ctx,
-		index,
-		service.loaders.PlaylistLoader,
-		service.loaders.PlaylistPermissionLoader,
-		service.queries.ListPlaylists,
-		service.playlistToSearchItem,
-	)
+	return indexObjectElastic[int, common.Episode](ctx, service.elasticClient, IndexEpisodes, *i, p, service.episodeToSearchItem)
 }
 
 func indexCollectionElastic[k comparable, t indexable[k]](
@@ -362,80 +205,6 @@ type indexable[k comparable] interface {
 	GetKey() k
 }
 
-func indexCollection[k comparable, t indexable[k]](
-	ctx context.Context,
-	index *search.Index,
-	loader *loaders.Loader[k, *t],
-	permissionLoader *loaders.Loader[k, *common.Permissions[k]],
-	factory func(context.Context) ([]t, error),
-	converter func(context.Context, t) (searchItem, error),
-) error {
-	items, err := factory(ctx)
-	if err != nil {
-		return err
-	}
-
-	ids := lo.Map(items, func(i t, _ int) k {
-		return i.GetKey()
-	})
-
-	permissionLoader.LoadMany(ctx, ids)()
-
-	var searchItems []searchObject
-	pushItems := func(force bool) error {
-		if len(searchItems) > 200 || (force && len(searchItems) > 0) {
-			_, err := index.SaveObjects(searchItems)
-			if err != nil {
-				return err
-			}
-			searchItems = []searchObject{}
-		}
-		return nil
-	}
-
-	for _, i := range items {
-		p := i
-		loader.Prime(ctx, p.GetKey(), &p)
-
-		item, err := converter(ctx, p)
-		if err != nil {
-			return err
-		}
-
-		perm, err := permissionLoader.Get(ctx, i.GetKey())
-		if err != nil {
-			return err
-		}
-
-		if perm != nil {
-			item.assignVisibility(perm.Availability)
-			item.assignRoles(perm.Roles)
-		}
-
-		searchItems = append(searchItems, item.toSearchObject())
-		err = pushItems(false)
-	}
-	return pushItems(true)
-}
-
-func indexObject[k comparable, t indexable[k]](
-	ctx context.Context,
-	service *Service,
-	obj t,
-	perms *common.Permissions[k],
-	converter func(context.Context, t) (searchItem, error),
-) error {
-	item, err := converter(ctx, obj)
-	if err != nil {
-		return err
-	}
-
-	item.assignVisibility(perms.Availability)
-	item.assignRoles(perms.Roles)
-	_, err = service.index.SaveObject(item)
-	return err
-}
-
 func indexObjectElastic[k comparable, t indexable[k]](
 	ctx context.Context,
 	elasticClient *elasticsearch.TypedClient,
@@ -473,11 +242,7 @@ func (service *Service) DeleteModel(ctx context.Context, collection string, key
 		return nil
 	}
 
-	_, err := service.index.DeleteObject(collection + "-" + key)
-	if err != nil {
-		return err
-	}
-
+	var err error
 	if index, ok := typeToIndexMap[collection]; ok {
 		_, err = service.elasticClient.Delete(index, collection+"-"+key).Do(ctx)
 	}