diff --git a/lib/jekyll-algolia.rb b/lib/jekyll-algolia.rb
index 4e21382..b6fa737 100644
--- a/lib/jekyll-algolia.rb
+++ b/lib/jekyll-algolia.rb
@@ -19,6 +19,8 @@ module Jekyll
     require 'jekyll/algolia/utils'
     require 'jekyll/algolia/version'
 
+    MissingCredentialsError = Class.new(StandardError)
+
     # Public: Init the Algolia module
     #
     # config - A hash of Jekyll config option (merge of _config.yml options and
@@ -33,7 +35,12 @@ module Jekyll
       config = Configurator.init(config).config
       @site = Jekyll::Algolia::Site.new(config)
 
-      exit 1 unless Configurator.assert_valid_credentials
+      unless Configurator.assert_valid_credentials
+        raise(
+          MissingCredentialsError,
+          "One or more credentials were not found for site at: #{@site.source}"
+        )
+      end
 
       Configurator.warn_of_deprecated_options
 
diff --git a/spec/jekyll-algolia_spec.rb b/spec/jekyll-algolia_spec.rb
index 3e198f6..9f03aff 100644
--- a/spec/jekyll-algolia_spec.rb
+++ b/spec/jekyll-algolia_spec.rb
@@ -57,7 +57,7 @@ describe(Jekyll::Algolia) do
           .and_return(false)
       end
 
-      it { is_expected.to raise_error SystemExit }
+      it { is_expected.to raise_error Jekyll::Algolia::MissingCredentialsError }
     end
   end
 
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index 7b83ce3..8b2eb78 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -14,7 +14,7 @@ require 'ostruct'
 
 RSpec.configure do |config|
   config.filter_run(focus: true)
-  config.fail_fast = true
+  config.fail_fast = false
   config.run_all_when_everything_filtered = true
   config.before do
     Jekyll::Algolia::Configurator.init