394 lines
13 KiB
Ruby
394 lines
13 KiB
Ruby
# frozen_string_literal: true
|
||
|
||
require 'algoliasearch'
|
||
require 'yaml'
|
||
require 'algolia_html_extractor'
|
||
|
||
# rubocop:disable Metrics/ModuleLength
|
||
module Jekyll
|
||
module Algolia
|
||
# Module to push records to Algolia and configure the index
|
||
module Indexer
|
||
include Jekyll::Algolia
|
||
|
||
# Public: Init the module
|
||
def self.init
|
||
::Algolia.init(
|
||
application_id: Configurator.application_id,
|
||
api_key: Configurator.api_key
|
||
)
|
||
index_name = Configurator.index_name
|
||
@index = ::Algolia::Index.new(index_name)
|
||
index_object_ids_name = Configurator.index_object_ids_name
|
||
@index_object_ids = ::Algolia::Index.new(index_object_ids_name)
|
||
|
||
set_user_agent
|
||
|
||
self
|
||
end
|
||
|
||
# Public: Returns the Algolia index object
|
||
def self.index
|
||
@index
|
||
end
|
||
|
||
# Public: Returns the Algolia index used to store object ids
|
||
def self.index_object_ids
|
||
@index_object_ids
|
||
end
|
||
|
||
# Public: Check if an index exists
|
||
#
|
||
# index - Index to check
|
||
#
|
||
# Note: there is no API endpoint to do that, so we try to get the settings
|
||
# instead, which will fail if the index does not exist
|
||
def self.index_exist?(index)
|
||
index.get_settings
|
||
true
|
||
rescue StandardError
|
||
false
|
||
end
|
||
|
||
# Public: Get the number of records in an index
|
||
#
|
||
# index - Index to check
|
||
#
|
||
# Note: We'll do an empty query search, to match everything, but we'll
|
||
# only return the objectID and one element, to get the shortest response
|
||
# possible. It will still contain the nbHits
|
||
def self.record_count(index)
|
||
index.search(
|
||
'',
|
||
attributesToRetrieve: 'objectID',
|
||
distinct: false,
|
||
hitsPerPage: 1
|
||
)['nbHits']
|
||
rescue StandardError
|
||
0
|
||
end
|
||
|
||
# Public: Set the User-Agent to send to the API
|
||
#
|
||
# Every integrations should follow the "YYY Integration" pattern, and
|
||
# every API client should follow the "Algolia for YYY" pattern. Even if
|
||
# each integration version is pinned to a specific API client version, we
|
||
# are explicit in defining it to help debug from the dashboard.
|
||
def self.set_user_agent
|
||
user_agent = [
|
||
"Jekyll Integration (#{VERSION})",
|
||
"Algolia for Ruby (#{::Algolia::VERSION})",
|
||
"Jekyll (#{::Jekyll::VERSION})",
|
||
"Ruby (#{RUBY_VERSION})"
|
||
].join('; ')
|
||
|
||
::Algolia.set_extra_header('User-Agent', user_agent)
|
||
end
|
||
|
||
# Public: Get an array of all object IDs stored in the main index
|
||
#
|
||
# Note: As this will be slow (grabbing them 1000 at a time), we display
|
||
# a progress bar.
|
||
def self.remote_object_ids_from_main_index
|
||
Logger.verbose("I:Inspecting existing records in index #{index.name}")
|
||
|
||
list = []
|
||
|
||
# As it might take some time, we display a progress bar
|
||
progress_bar = ProgressBar.create(
|
||
total: record_count(index),
|
||
format: 'Inspecting existing records (%j%%) |%B|'
|
||
)
|
||
begin
|
||
index.browse(
|
||
attributesToRetrieve: 'objectID',
|
||
hitsPerPage: 1000
|
||
) do |hit|
|
||
list << hit['objectID']
|
||
progress_bar.increment
|
||
end
|
||
rescue StandardError
|
||
return []
|
||
end
|
||
|
||
list.sort
|
||
end
|
||
|
||
# Public: Get an array of all the object ids, stored in a dedicated
|
||
# index
|
||
#
|
||
# Note: This will be very fast. Each record contain 100 object id, so it
|
||
# will fit in one call each time.
|
||
def self.remote_object_ids_from_dedicated_index
|
||
list = []
|
||
begin
|
||
index_object_ids.browse(
|
||
attributesToRetrieve: 'content',
|
||
hitsPerPage: 1000
|
||
) do |hit|
|
||
list += hit['content']
|
||
end
|
||
rescue StandardError
|
||
return []
|
||
end
|
||
|
||
list.sort
|
||
end
|
||
|
||
# Public: Returns an array of all the objectIDs in the index
|
||
#
|
||
# Note: We use a dedicated index to store the objectIDs for faster
|
||
# browsing, but if the index does not exist we read the main index.
|
||
def self.remote_object_ids
|
||
Logger.log('I:Getting list of existing records')
|
||
|
||
# Main index empty, the list is empty no matter what (we don't use the
|
||
# dedicated index in that case)
|
||
return [] if record_count(index).zero?
|
||
|
||
# Fast version, using the dedicated index
|
||
has_object_id_index = index_exist?(index_object_ids)
|
||
return remote_object_ids_from_dedicated_index if has_object_id_index
|
||
|
||
# Slow version, browsing the full index
|
||
remote_object_ids_from_main_index
|
||
end
|
||
|
||
# Public: Returns an array of the local objectIDs
|
||
#
|
||
# records - Array of all local records
|
||
def self.local_object_ids(records)
|
||
records.map { |record| record[:objectID] }.compact.sort
|
||
end
|
||
|
||
# Public: Update records of the index
|
||
#
|
||
# records - All records extracted from Jekyll
|
||
#
|
||
# Note: All operations will be done in one batch, assuring an atomic
|
||
# update
|
||
# Does nothing in dry run mode
|
||
def self.update_records(records)
|
||
# Getting list of objectID in remote and locally
|
||
remote_ids = remote_object_ids
|
||
local_ids = local_object_ids(records)
|
||
|
||
# Making a diff, to see what to add and what to delete
|
||
ids_to_delete = remote_ids - local_ids
|
||
ids_to_add = local_ids - remote_ids
|
||
|
||
# What changes should we do to the indexes?
|
||
has_records_to_update = !ids_to_delete.empty? || !ids_to_add.empty?
|
||
has_object_id_index = index_exist?(index_object_ids)
|
||
|
||
# Stop if nothing to change
|
||
if !has_records_to_update && has_object_id_index
|
||
Logger.log('I:Content is already up to date.')
|
||
return
|
||
end
|
||
|
||
# We group all operations into one batch
|
||
operations = []
|
||
|
||
# We update records only if there are records to update
|
||
if has_records_to_update
|
||
Logger.log("I:Updating records in index #{index.name}...")
|
||
Logger.log("I:Records to delete: #{ids_to_delete.length}")
|
||
Logger.log("I:Records to add: #{ids_to_add.length}")
|
||
|
||
# Transforming ids into real records to add
|
||
records_by_id = Hash[records.map { |r| [r[:objectID], r] }]
|
||
records_to_add = ids_to_add.map { |id| records_by_id[id] }
|
||
|
||
# Deletion operations come first, to avoid hitting an overquota too
|
||
# soon if it can be avoided
|
||
ids_to_delete.each do |object_id|
|
||
operations << {
|
||
action: 'deleteObject', indexName: index.name,
|
||
body: { objectID: object_id }
|
||
}
|
||
end
|
||
# Then we add the new records
|
||
operations += records_to_add.map do |new_record|
|
||
{ action: 'addObject', indexName: index.name, body: new_record }
|
||
end
|
||
end
|
||
|
||
# We update the dedicated index everytime we update records, but we also
|
||
# create it if it does not exist
|
||
should_update_object_id_index = has_records_to_update ||
|
||
!has_object_id_index
|
||
if should_update_object_id_index
|
||
operations << { action: 'clear', indexName: index_object_ids.name }
|
||
local_ids.each_slice(100).each do |ids|
|
||
operations << {
|
||
action: 'addObject', indexName: index_object_ids.name,
|
||
body: { content: ids }
|
||
}
|
||
end
|
||
end
|
||
|
||
execute_operations(operations)
|
||
end
|
||
|
||
# Public: Execute a serie of operations in a batch
|
||
#
|
||
# operations - Operations to batch
|
||
#
|
||
# Note: Will split the batch in several calls if too big, and will display
|
||
# a progress bar if this happens
|
||
def self.execute_operations(operations)
|
||
return if Configurator.dry_run?
|
||
return if operations.empty?
|
||
|
||
# Run the batches in slices if they are too large
|
||
batch_size = Configurator.algolia('indexing_batch_size')
|
||
slices = operations.each_slice(batch_size).to_a
|
||
|
||
should_have_progress_bar = (slices.length > 1)
|
||
if should_have_progress_bar
|
||
progress_bar = ProgressBar.create(
|
||
total: slices.length,
|
||
format: 'Updating index (%j%%) |%B|'
|
||
)
|
||
end
|
||
|
||
slices.each do |slice|
|
||
begin
|
||
::Algolia.batch!(slice)
|
||
|
||
progress_bar.increment if should_have_progress_bar
|
||
rescue StandardError => error
|
||
ErrorHandler.stop(error, operations: slice)
|
||
end
|
||
end
|
||
end
|
||
|
||
# Public: Get a unique settingID for the current settings
|
||
#
|
||
# The settingID is generated as a hash of the current settings. As it will
|
||
# be stored in the userData key of the resulting config, we exclude that
|
||
# key from the hashing.
|
||
def self.local_setting_id
|
||
settings = Configurator.settings
|
||
settings.delete('userData')
|
||
AlgoliaHTMLExtractor.uuid(settings)
|
||
end
|
||
|
||
# Public: Get the settings of the remote index
|
||
#
|
||
# In case the index is not accessible, it will return nil
|
||
def self.remote_settings
|
||
index.get_settings
|
||
rescue StandardError
|
||
nil
|
||
end
|
||
|
||
# Public: Smart update of the settings of the index
|
||
#
|
||
# This will first compare the settings about to be pushed with the
|
||
# settings already pushed. It will compare userData.settingID for that.
|
||
# If the settingID is the same, we don't push as this won't change
|
||
# anything. We will still check if the remote config seem to have been
|
||
# manually altered though, and warn the user that this is not the
|
||
# preferred way of doing so.
|
||
#
|
||
# If the settingID are not matching, it means our config is different, so
|
||
# we push it, overriding the settingID for next push.
|
||
def self.update_settings
|
||
current_remote_settings = remote_settings || {}
|
||
remote_setting_id = current_remote_settings.dig('userData', 'settingID')
|
||
|
||
settings = Configurator.settings
|
||
setting_id = local_setting_id
|
||
|
||
are_settings_forced = Configurator.force_settings?
|
||
|
||
# The config we're about to push is the same we pushed previously. We
|
||
# won't push again.
|
||
if setting_id == remote_setting_id && !are_settings_forced
|
||
Logger.log('I:Settings are already up to date.')
|
||
# Check if remote config has been changed outside of the plugin, so we
|
||
# can warn users that they should not alter their config from outside
|
||
# of the plugin.
|
||
current_remote_settings.delete('userData')
|
||
changed_keys = Utils.diff_keys(settings, current_remote_settings)
|
||
unless changed_keys.nil?
|
||
warn_of_manual_dashboard_editing(changed_keys)
|
||
end
|
||
|
||
return
|
||
end
|
||
|
||
# Settings have changed, we push them
|
||
settings['userData'] = {
|
||
'settingID' => setting_id,
|
||
'pluginVersion' => VERSION
|
||
}
|
||
|
||
Logger.log("I:Updating settings of index #{index.name}")
|
||
return if Configurator.dry_run?
|
||
set_settings(settings)
|
||
end
|
||
|
||
# Public: Set new settings to an index
|
||
#
|
||
# Will dispatch to the error handler if it fails
|
||
# rubocop:disable Naming/AccessorMethodName
|
||
def self.set_settings(settings)
|
||
index.set_settings!(settings)
|
||
rescue StandardError => error
|
||
ErrorHandler.stop(error, settings: settings)
|
||
end
|
||
# rubocop:enable Naming/AccessorMethodName
|
||
|
||
# Public: Warn users that they have some settings manually configured in
|
||
# their dashboard
|
||
#
|
||
# When users change some settings in their dashboard, those settings might
|
||
# get overwritten by the pluging. We can't prevent that, but we can warn
|
||
# them when we detect they changed something.
|
||
def self.warn_of_manual_dashboard_editing(changed_keys)
|
||
# Transform the hash into readable YAML
|
||
yaml_lines = changed_keys
|
||
.to_yaml(indentation: 2)
|
||
.split("\n")[1..-1]
|
||
yaml_lines.map! do |line|
|
||
line = line.gsub(/^ */) { |spaces| ' ' * spaces.length }
|
||
line = line.gsub('- ', ' - ')
|
||
"W: #{line}"
|
||
end
|
||
Logger.known_message(
|
||
'settings_manually_edited',
|
||
settings: yaml_lines.join("\n"),
|
||
index_name: Configurator.index_name
|
||
)
|
||
end
|
||
|
||
# Public: Push all records to Algolia and configure the index
|
||
#
|
||
# records - Records to push
|
||
def self.run(records)
|
||
init
|
||
|
||
# Indexing zero record is surely a misconfiguration
|
||
if records.length.zero?
|
||
files_to_exclude = Configurator.algolia('files_to_exclude').join(', ')
|
||
Logger.known_message(
|
||
'no_records_found',
|
||
'files_to_exclude' => files_to_exclude,
|
||
'nodes_to_index' => Configurator.algolia('nodes_to_index')
|
||
)
|
||
exit 1
|
||
end
|
||
|
||
update_settings
|
||
update_records(records)
|
||
|
||
Logger.log('I:✔ Indexing complete')
|
||
end
|
||
end
|
||
end
|
||
end
|
||
# rubocop:enable Metrics/ModuleLength
|